Report mean ATE alongside benchmark AUC

README.md

@@ -531,19 +531,21 @@ python -m vidmap.run_for_benchmark \
 <summary>[Results - click to expand]</summary>
 
 WATE-AUC with late principal point refinement, at 5% error thresholds (%, higher
-is better). Results are averaged equally across eligible sequences:
+is better). Results are averaged equally across eligible sequences. ATE is the mean
+per-sequence translation RMSE after global Sim(3) alignment (meters, lower is
+better), evaluated at GT-supported estimated poses:
 
-| Dataset | 10 m @ 0.5 m | 25 m @ 1.25 m | 50 m @ 2.5 m | 100 m @ 5 m | Full @ 5% |
-| :--- | ---: | ---: | ---: | ---: | ---: |
-| LaMAR (sample-300, 50 sequences) | 92.43 | 93.08 | 91.79 | 91.03 | 90.41 |
-| CroCoDL (64 sequences) | 95.72 | 97.27 | 97.39 | 96.56 | 98.21 |
+| Dataset | ATE (m) ↓ | 10 m @ 0.5 m | 25 m @ 1.25 m | 50 m @ 2.5 m | 100 m @ 5 m | Full @ 5% |
+| :--- | ---: | ---: | ---: | ---: | ---: | ---: |
+| LaMAR (sample-300, 50 sequences) | 0.878 | 92.43 | 93.08 | 91.79 | 91.03 | 90.41 |
+| CroCoDL (64 sequences) | 0.233 | 95.72 | 97.27 | 97.39 | 96.56 | 98.21 |
 
 Pose AUC, averaged equally across sequences (%, higher is better):
 
-| Dataset | 5 cm | 10 cm | 50 cm | 1 m | 10 m |
-| :--- | ---: | ---: | ---: | ---: | ---: |
-| EuRoC (11 sequences) | 28.44 | 59.74 | 91.91 | 95.95 | 99.60 |
-| ETH3D-SLAM (55 sequences) | 80.32 | 87.85 | 95.86 | 97.35 | 99.28 |
+| Dataset | ATE (m) ↓ | 5 cm | 10 cm | 50 cm | 1 m | 10 m |
+| :--- | ---: | ---: | ---: | ---: | ---: | ---: |
+| EuRoC (11 sequences) | 0.044 | 28.44 | 59.74 | 91.91 | 95.95 | 99.60 |
+| ETH3D-SLAM (55 sequences) | 0.268 | 80.32 | 87.85 | 95.86 | 97.35 | 99.28 |
 
 </details>
 

vidmap/benchmark/report.py

@@ -43,9 +43,10 @@ def build_evaluation_parser():
 
     add_evaluation_arguments(parser)
     parser.add_argument(
-        "--only-wate-auc",
-        action="store_true",
-        help="Only print WATE-AUC tables/CSV; suppress WATE median/mean/count and WRRE output.",
+        "--metrics",
+        nargs="+",
+        choices=("ate", "pose-auc", "wate-auc"),
+        help="Metrics to print in text and CSV (default: ATE and the dataset's AUC metric).",
     )
     return parser
 
@@ -147,13 +148,15 @@ def __init__(
         prints: dict,
         colls: dict,
         wate_colls: dict,
+        ate_colls: dict,
     ):
         self.args = args
         self.modes = modes
         self.scenes = scenes
         self.prints = prints
         self.colls = colls
         self.wate_colls = wate_colls
+        self.ate_colls = ate_colls
 
     @classmethod
     def collect(
@@ -180,6 +183,7 @@ def collect(
         prints = {}
         colls = {}
         wate_colls = {}
+        ate_colls = {}
         wate_auc_full_thresholds = sorted({percent / 100.0 for percent in (args.wate_auc_percents or [])})
         for mode in modes:
             args.mode = mode
@@ -203,7 +207,7 @@ def collect(
                 output_name=args.name,
                 run_options=run_options,
             )
-            out, counts, _, out_wate = result_evaluator.evaluate_saved()
+            out, counts, _, out_wate, ate_colls[mode] = result_evaluator.evaluate_saved()
             prints[mode] = {}
             colls[mode] = {}
             for scene, _AUCs in out.items():
@@ -261,6 +265,7 @@ def collect(
             prints=prints,
             colls=colls,
             wate_colls=wate_colls,
+            ate_colls=ate_colls,
         )
 
     def render(self) -> None:
@@ -275,8 +280,10 @@ def render(self) -> None:
         from vidmap.datasets.registry import get_dataset_spec
 
         default_metric = get_dataset_spec(args.dataset).default_evaluation_metric
-        show_pose_auc = default_metric == "pose_auc" and not args.only_wate_auc
-        show_wate_auc = default_metric == "wate_auc" or args.only_wate_auc
+        metrics = args.metrics if args.metrics is not None else ("ate", default_metric.replace("_", "-"))
+        show_ate = "ate" in metrics
+        show_pose_auc = "pose-auc" in metrics
+        show_wate_auc = "wate-auc" in metrics
         wate_alignment_label = "Sim(3)-window scale-aligned"
 
         for mode in modes:
@@ -303,6 +310,16 @@ def render(self) -> None:
                 failed_sum = np.sum([counts["failed"] for values, counts in coll.values()])
                 print(f"     recs total: {comp_sum}/{comp_sum + failed_sum}")
 
+            if show_ate:
+                print("  ATE RMSE (m, mean across trajectories):")
+                for scene in scenes:
+                    values = self.ate_colls[mode][scene]
+                    value = f"{np.mean(values):.3f}" if values else "missing"
+                    print(f"     {scene}: {value}    trajectories: {len(values)}")
+                values = [value for scene in scenes for value in self.ate_colls[mode][scene]]
+                value = f"{np.mean(values):.3f}" if values else "missing"
+                print(f"     total: {value}    trajectories: {len(values)}")
+
             # Windowed ATE
             wcoll = wate_colls[mode]
             # Find window sizes present across scenes (e.g. 10, 25, 50, 100)
@@ -401,6 +418,18 @@ def render(self) -> None:
                             auc_str = " / ".join(f"{v:.1f}" for v in values)
                         print(sep.join([scene, auc_str]))
 
+        if show_ate:
+            print("\n<!-- ate csv -->")
+            print(sep.join(["Mode", "Scene", "ATE RMSE (m)", "Trajectories"]))
+            for mode in modes:
+                for scene in scenes:
+                    values = self.ate_colls[mode][scene]
+                    value = f"{np.mean(values):.3f}" if values else ""
+                    print(sep.join([mode, scene, value, str(len(values))]))
+                values = [value for scene in scenes for value in self.ate_colls[mode][scene]]
+                value = f"{np.mean(values):.3f}" if values else ""
+                print(sep.join([mode, "total", value, str(len(values))]))
+
         # WATE CSV (separate from AUC/Recall)
         all_window_sizes_csv = sorted(
             set(

vidmap/benchmark/result_evaluation.py

@@ -52,7 +52,7 @@ def _report_group(self, case):
             return f"{case.scene}-{case.recording}"
         return case.scene
 
-    def evaluate_case(self, case, scene_parser, outputs, recalls, wates):
+    def evaluate_case(self, case, scene_parser, outputs, recalls, wates, ates):
         """Load, evaluate, and aggregate one persisted benchmark reconstruction."""
         group = self._report_group(case)
         try:
@@ -86,6 +86,8 @@ def evaluate_case(self, case, scene_parser, outputs, recalls, wates):
             )
 
             summary = evaluation.summarize(verbose=logger.isEnabledFor(logging.INFO))
+            if evaluation.ate is not None:
+                ates[group].append(evaluation.ate["rmse"])
             outputs[group][case.testset_type].append(summary["all"])
             if "recall" in summary:
                 recalls[group][case.testset_type].append(summary["recall"])
@@ -111,6 +113,7 @@ def evaluate_saved(self):
             )
         outputs = {group: defaultdict(list) for group in groups}
         recalls = {group: defaultdict(list) for group in groups}
+        ates = {group: [] for group in groups}
         wates = {group: defaultdict(lambda: defaultdict(list)) for group in groups}
         counts = {group: {"completed": 0, "failed": 0} for group in groups}
         for case, scene_parser, _ in tqdm(
@@ -124,6 +127,6 @@ def evaluate_saved(self):
                 testset=case.testset_desc,
                 config=self.conf.name,
             ):
-                ok, group = self.evaluate_case(case, scene_parser, outputs, recalls, wates)
+                ok, group = self.evaluate_case(case, scene_parser, outputs, recalls, wates, ates)
                 counts[group]["completed" if ok else "failed"] += 1
-        return outputs, counts, recalls, wates
+        return outputs, counts, recalls, wates, ates

vidmap/benchmark/trajectory.py

@@ -456,25 +456,19 @@ def evaluate(
     ) -> AbsoluteTrajectoryResult:
         """Evaluate cam0, matching estimates to GT by image name.
 
-        With align_in_place, also align the full reconstruction for saving and
-        return its cam0 trajectories and ATE using the paper's two-pass alignment.
+        Return cam0 trajectories and ATE using the paper's two-pass alignment.
+        With align_in_place, also align the full reconstruction for saving.
         Otherwise, leave both input reconstructions unchanged.
         """
         full_reconstruction = estimated_reconstruction
         estimated_reconstruction, ground_truth_reconstruction = _prepare_evaluation_pair(
             estimated_reconstruction, ground_truth_reconstruction, self.reference_from_gt
         )
-        ate = estimated_trajectory = ground_truth_trajectory = None
         if align_in_place:
             _align_reconstruction_for_evaluation(
                 estimated_reconstruction, ground_truth_reconstruction, full_reconstruction=full_reconstruction
             )
-            estimated_trajectory = reconstruction_to_evo_trajectory(estimated_reconstruction)
-            supported_ground_truth = remap_poses_to_timeline(
-                estimated_reconstruction, ground_truth_reconstruction, interpolate_missing_poses=False
-            ).reconstruction
-            ground_truth_trajectory = reconstruction_to_evo_trajectory(supported_ground_truth)
-            ate = compute_ate_statistics(ground_truth_trajectory, estimated_trajectory)
+            ate_reconstruction = estimated_reconstruction
         num_images = estimated_reconstruction.num_images()
         num_registered_images = estimated_reconstruction.num_reg_images()
         estimated_reconstruction, _ = align_reconstruction_to_reference_sequence(
@@ -506,6 +500,18 @@ def evaluate(
 
         data["results"]["scale"] = self.options.gt_scale
         data["success"] = True
+        if not align_in_place:
+            ate_reconstruction, _ = align_reconstruction_to_reference_sequence(
+                estimated_reconstruction, ground_truth_reconstruction
+            )
+        supported_ground_truth = remap_poses_to_timeline(
+            ate_reconstruction, ground_truth_reconstruction, interpolate_missing_poses=False
+        ).reconstruction
+        ate = estimated_trajectory = ground_truth_trajectory = None
+        if supported_ground_truth.num_reg_images():
+            estimated_trajectory = reconstruction_to_evo_trajectory(ate_reconstruction)
+            ground_truth_trajectory = reconstruction_to_evo_trajectory(supported_ground_truth)
+            ate = compute_ate_statistics(ground_truth_trajectory, estimated_trajectory)
         return AbsoluteTrajectoryResult(
             options=self.options,
             data=data,