From c2e4ec9c88b66ee8db216a5495672c6598cd7238 Mon Sep 17 00:00:00 2001 From: Zador-Pataki Date: Wed, 4 Nov 2026 02:50:38 +0100 Subject: [PATCH] Report mean ATE alongside benchmark AUC --- README.md | 20 +++++++------ vidmap/benchmark/report.py | 41 +++++++++++++++++++++++---- vidmap/benchmark/result_evaluation.py | 9 ++++-- vidmap/benchmark/trajectory.py | 24 ++++++++++------ 4 files changed, 67 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index fd865fd..ed2e6af 100644 --- a/README.md +++ b/README.md @@ -531,19 +531,21 @@ python -m vidmap.run_for_benchmark \ [Results - click to expand] WATE-AUC with late principal point refinement, at 5% error thresholds (%, higher -is better). Results are averaged equally across eligible sequences: +is better). Results are averaged equally across eligible sequences. ATE is the mean +per-sequence translation RMSE after global Sim(3) alignment (meters, lower is +better), evaluated at GT-supported estimated poses: -| Dataset | 10 m @ 0.5 m | 25 m @ 1.25 m | 50 m @ 2.5 m | 100 m @ 5 m | Full @ 5% | -| :--- | ---: | ---: | ---: | ---: | ---: | -| LaMAR (sample-300, 50 sequences) | 92.43 | 93.08 | 91.79 | 91.03 | 90.41 | -| CroCoDL (64 sequences) | 95.72 | 97.27 | 97.39 | 96.56 | 98.21 | +| Dataset | ATE (m) ↓ | 10 m @ 0.5 m | 25 m @ 1.25 m | 50 m @ 2.5 m | 100 m @ 5 m | Full @ 5% | +| :--- | ---: | ---: | ---: | ---: | ---: | ---: | +| LaMAR (sample-300, 50 sequences) | 0.878 | 92.43 | 93.08 | 91.79 | 91.03 | 90.41 | +| CroCoDL (64 sequences) | 0.233 | 95.72 | 97.27 | 97.39 | 96.56 | 98.21 | Pose AUC, averaged equally across sequences (%, higher is better): -| Dataset | 5 cm | 10 cm | 50 cm | 1 m | 10 m | -| :--- | ---: | ---: | ---: | ---: | ---: | -| EuRoC (11 sequences) | 28.44 | 59.74 | 91.91 | 95.95 | 99.60 | -| ETH3D-SLAM (55 sequences) | 80.32 | 87.85 | 95.86 | 97.35 | 99.28 | +| Dataset | ATE (m) ↓ | 5 cm | 10 cm | 50 cm | 1 m | 10 m | +| :--- | ---: | ---: | ---: | ---: | ---: | ---: | +| EuRoC (11 sequences) | 0.044 | 28.44 | 59.74 | 91.91 | 95.95 | 99.60 | +| ETH3D-SLAM (55 sequences) | 0.268 | 80.32 | 87.85 | 95.86 | 97.35 | 99.28 | diff --git a/vidmap/benchmark/report.py b/vidmap/benchmark/report.py index 5ead036..8f08f1c 100644 --- a/vidmap/benchmark/report.py +++ b/vidmap/benchmark/report.py @@ -43,9 +43,10 @@ def build_evaluation_parser(): add_evaluation_arguments(parser) parser.add_argument( - "--only-wate-auc", - action="store_true", - help="Only print WATE-AUC tables/CSV; suppress WATE median/mean/count and WRRE output.", + "--metrics", + nargs="+", + choices=("ate", "pose-auc", "wate-auc"), + help="Metrics to print in text and CSV (default: ATE and the dataset's AUC metric).", ) return parser @@ -147,6 +148,7 @@ def __init__( prints: dict, colls: dict, wate_colls: dict, + ate_colls: dict, ): self.args = args self.modes = modes @@ -154,6 +156,7 @@ def __init__( self.prints = prints self.colls = colls self.wate_colls = wate_colls + self.ate_colls = ate_colls @classmethod def collect( @@ -180,6 +183,7 @@ def collect( prints = {} colls = {} wate_colls = {} + ate_colls = {} wate_auc_full_thresholds = sorted({percent / 100.0 for percent in (args.wate_auc_percents or [])}) for mode in modes: args.mode = mode @@ -203,7 +207,7 @@ def collect( output_name=args.name, run_options=run_options, ) - out, counts, _, out_wate = result_evaluator.evaluate_saved() + out, counts, _, out_wate, ate_colls[mode] = result_evaluator.evaluate_saved() prints[mode] = {} colls[mode] = {} for scene, _AUCs in out.items(): @@ -261,6 +265,7 @@ def collect( prints=prints, colls=colls, wate_colls=wate_colls, + ate_colls=ate_colls, ) def render(self) -> None: @@ -275,8 +280,10 @@ def render(self) -> None: from vidmap.datasets.registry import get_dataset_spec default_metric = get_dataset_spec(args.dataset).default_evaluation_metric - show_pose_auc = default_metric == "pose_auc" and not args.only_wate_auc - show_wate_auc = default_metric == "wate_auc" or args.only_wate_auc + metrics = args.metrics if args.metrics is not None else ("ate", default_metric.replace("_", "-")) + show_ate = "ate" in metrics + show_pose_auc = "pose-auc" in metrics + show_wate_auc = "wate-auc" in metrics wate_alignment_label = "Sim(3)-window scale-aligned" for mode in modes: @@ -303,6 +310,16 @@ def render(self) -> None: failed_sum = np.sum([counts["failed"] for values, counts in coll.values()]) print(f" recs total: {comp_sum}/{comp_sum + failed_sum}") + if show_ate: + print(" ATE RMSE (m, mean across trajectories):") + for scene in scenes: + values = self.ate_colls[mode][scene] + value = f"{np.mean(values):.3f}" if values else "missing" + print(f" {scene}: {value} trajectories: {len(values)}") + values = [value for scene in scenes for value in self.ate_colls[mode][scene]] + value = f"{np.mean(values):.3f}" if values else "missing" + print(f" total: {value} trajectories: {len(values)}") + # Windowed ATE wcoll = wate_colls[mode] # Find window sizes present across scenes (e.g. 10, 25, 50, 100) @@ -401,6 +418,18 @@ def render(self) -> None: auc_str = " / ".join(f"{v:.1f}" for v in values) print(sep.join([scene, auc_str])) + if show_ate: + print("\n") + print(sep.join(["Mode", "Scene", "ATE RMSE (m)", "Trajectories"])) + for mode in modes: + for scene in scenes: + values = self.ate_colls[mode][scene] + value = f"{np.mean(values):.3f}" if values else "" + print(sep.join([mode, scene, value, str(len(values))])) + values = [value for scene in scenes for value in self.ate_colls[mode][scene]] + value = f"{np.mean(values):.3f}" if values else "" + print(sep.join([mode, "total", value, str(len(values))])) + # WATE CSV (separate from AUC/Recall) all_window_sizes_csv = sorted( set( diff --git a/vidmap/benchmark/result_evaluation.py b/vidmap/benchmark/result_evaluation.py index 116cf7f..1b51d89 100644 --- a/vidmap/benchmark/result_evaluation.py +++ b/vidmap/benchmark/result_evaluation.py @@ -52,7 +52,7 @@ def _report_group(self, case): return f"{case.scene}-{case.recording}" return case.scene - def evaluate_case(self, case, scene_parser, outputs, recalls, wates): + def evaluate_case(self, case, scene_parser, outputs, recalls, wates, ates): """Load, evaluate, and aggregate one persisted benchmark reconstruction.""" group = self._report_group(case) try: @@ -86,6 +86,8 @@ def evaluate_case(self, case, scene_parser, outputs, recalls, wates): ) summary = evaluation.summarize(verbose=logger.isEnabledFor(logging.INFO)) + if evaluation.ate is not None: + ates[group].append(evaluation.ate["rmse"]) outputs[group][case.testset_type].append(summary["all"]) if "recall" in summary: recalls[group][case.testset_type].append(summary["recall"]) @@ -111,6 +113,7 @@ def evaluate_saved(self): ) outputs = {group: defaultdict(list) for group in groups} recalls = {group: defaultdict(list) for group in groups} + ates = {group: [] for group in groups} wates = {group: defaultdict(lambda: defaultdict(list)) for group in groups} counts = {group: {"completed": 0, "failed": 0} for group in groups} for case, scene_parser, _ in tqdm( @@ -124,6 +127,6 @@ def evaluate_saved(self): testset=case.testset_desc, config=self.conf.name, ): - ok, group = self.evaluate_case(case, scene_parser, outputs, recalls, wates) + ok, group = self.evaluate_case(case, scene_parser, outputs, recalls, wates, ates) counts[group]["completed" if ok else "failed"] += 1 - return outputs, counts, recalls, wates + return outputs, counts, recalls, wates, ates diff --git a/vidmap/benchmark/trajectory.py b/vidmap/benchmark/trajectory.py index 33e0088..3dae105 100644 --- a/vidmap/benchmark/trajectory.py +++ b/vidmap/benchmark/trajectory.py @@ -456,25 +456,19 @@ def evaluate( ) -> AbsoluteTrajectoryResult: """Evaluate cam0, matching estimates to GT by image name. - With align_in_place, also align the full reconstruction for saving and - return its cam0 trajectories and ATE using the paper's two-pass alignment. + Return cam0 trajectories and ATE using the paper's two-pass alignment. + With align_in_place, also align the full reconstruction for saving. Otherwise, leave both input reconstructions unchanged. """ full_reconstruction = estimated_reconstruction estimated_reconstruction, ground_truth_reconstruction = _prepare_evaluation_pair( estimated_reconstruction, ground_truth_reconstruction, self.reference_from_gt ) - ate = estimated_trajectory = ground_truth_trajectory = None if align_in_place: _align_reconstruction_for_evaluation( estimated_reconstruction, ground_truth_reconstruction, full_reconstruction=full_reconstruction ) - estimated_trajectory = reconstruction_to_evo_trajectory(estimated_reconstruction) - supported_ground_truth = remap_poses_to_timeline( - estimated_reconstruction, ground_truth_reconstruction, interpolate_missing_poses=False - ).reconstruction - ground_truth_trajectory = reconstruction_to_evo_trajectory(supported_ground_truth) - ate = compute_ate_statistics(ground_truth_trajectory, estimated_trajectory) + ate_reconstruction = estimated_reconstruction num_images = estimated_reconstruction.num_images() num_registered_images = estimated_reconstruction.num_reg_images() estimated_reconstruction, _ = align_reconstruction_to_reference_sequence( @@ -506,6 +500,18 @@ def evaluate( data["results"]["scale"] = self.options.gt_scale data["success"] = True + if not align_in_place: + ate_reconstruction, _ = align_reconstruction_to_reference_sequence( + estimated_reconstruction, ground_truth_reconstruction + ) + supported_ground_truth = remap_poses_to_timeline( + ate_reconstruction, ground_truth_reconstruction, interpolate_missing_poses=False + ).reconstruction + ate = estimated_trajectory = ground_truth_trajectory = None + if supported_ground_truth.num_reg_images(): + estimated_trajectory = reconstruction_to_evo_trajectory(ate_reconstruction) + ground_truth_trajectory = reconstruction_to_evo_trajectory(supported_ground_truth) + ate = compute_ate_statistics(ground_truth_trajectory, estimated_trajectory) return AbsoluteTrajectoryResult( options=self.options, data=data,