Fix ETH3D ground-truth preparation and evaluation

README.md

@@ -527,7 +527,7 @@ Pose AUC, averaged equally across sequences (%, higher is better):
 | Dataset | 5 cm | 10 cm | 50 cm | 1 m | 10 m |
 | :--- | ---: | ---: | ---: | ---: | ---: |
 | EuRoC (11 sequences) | 28.86 | 60.02 | 91.96 | 95.98 | 99.60 |
-| ETH3D-SLAM (55 sequences) | 63.96 | 74.59 | 89.19 | 93.25 | 98.91 |
+| ETH3D-SLAM (55 sequences) | 78.48 | 85.74 | 94.39 | 96.61 | 99.52 |
 
 </details>
 

vidmap/benchmark/trajectory.py

@@ -409,6 +409,7 @@ def evaluate(
         remap = remap_poses_to_timeline(
             ground_truth_reconstruction,
             estimated_reconstruction,
+            # Full-GT evaluation must retain missing estimates as failures.
             prefer_exact_keyframes=True,
         )
         data = {
@@ -423,6 +424,8 @@ def evaluate(
         ground_truth = ground_truth_reconstruction
         if remap.target_subset is not None and not self.options.eval_on_full_gt_timeline:
             ground_truth = remap.target_subset
+        if ground_truth.num_reg_images() == 0:
+            raise ValueError("No ground-truth-supported poses available for evaluation")
         data |= self.evaluate_aligned_reconstructions(remap.reconstruction, ground_truth)
 
         data["results"]["scale"] = self.options.gt_scale

vidmap/benchmarking.py

@@ -199,7 +199,7 @@ def run(
         align_reconstruction_for_evaluation(estimated_reconstruction, scene_parser.rec)
         estimated_trajectory = reconstruction_to_evo_trajectory(estimated_reconstruction)
         aligned_ground_truth_reconstruction = remap_poses_to_timeline(
-            estimated_reconstruction, scene_parser.rec
+            estimated_reconstruction, scene_parser.rec, interpolate_missing_poses=False
         ).reconstruction
         ground_truth_trajectory = reconstruction_to_evo_trajectory(aligned_ground_truth_reconstruction)
         absolute_trajectory_error = compute_ate_statistics(ground_truth_trajectory, estimated_trajectory)

vidmap/datasets/base.py

@@ -271,7 +271,9 @@ def __init__(self, layout: DatasetLayout, scene: str) -> None:
             if unknown:
                 raise ValueError(f"Canonical unposed-image sidecar references unknown images: {sorted(unknown)}")
             unposed_names.update(sidecar_names)
-        for frame_id in sorted({images_by_name[name].frame_id for name in unposed_names}):
+        for frame_id in sorted(
+            {images_by_name[name].frame_id for name in unposed_names if images_by_name[name].has_pose}
+        ):
             self.rec.deregister_frame(frame_id)
 
     def camera(self, imid: int) -> pycolmap.Camera:

vidmap/datasets/eth3d_slam.py

@@ -2,6 +2,8 @@
 
 from vidmap.datasets.video import VideoDatasetManifest, VideoSequence
 
+ETH3D_SLAM_PREPARATION_VERSION = 3
+
 ETH3D_SLAM_SCENES = (
     "cables_1",
     "cables_2",

vidmap/datasets/prepare/eth3d_slam.py

@@ -17,16 +17,14 @@
 from PIL import Image
 
 from vidmap.datasets.base import CANONICAL_UNPOSED_IMAGES_FILENAME, DatasetLayout
-from vidmap.datasets.eth3d_slam import ETH3D_SLAM_MANIFEST
+from vidmap.datasets.eth3d_slam import ETH3D_SLAM_MANIFEST, ETH3D_SLAM_PREPARATION_VERSION
 from vidmap.datasets.layouts import get_dataset_layout
 from vidmap.datasets.prepare import filesystem as prep_fs
-from vidmap.datasets.prepare.timestamp_matching import assign_images_to_timestamp_ids
 from vidmap.datasets.prepare.video_common import resumable_http_download, safe_extract_zip
 from vidmap.datasets.resources import dataset_asset
 
 _URL_TEMPLATE = "https://www.eth3d.net/data/slam/datasets/{scene}_mono.zip"
 _SOURCE_RECORD = ".source_archive.json"
-ETH3D_SLAM_PREPARATION_VERSION = 2
 ETH3D_SLAM_LAYOUT = get_dataset_layout("eth3d_slam")
 
 
@@ -134,16 +132,12 @@ def _scene_is_complete(layout: DatasetLayout, scene: str) -> bool:
         archive = _source_archive(scene)
     except ValueError:
         archive = None
-    expected_unposed = None
-    if archive is not None:
-        expected_unposed = set() if archive.gt_available else names
     return (
         bool(reconstruction.cameras)
         and bool(image_ids)
         and testsets == {"0": image_ids}
         and unposed is not None
         and set(unposed) <= names
-        and (expected_unposed is None or set(unposed) == expected_unposed)
         and isinstance(preparation, Mapping)
         and preparation.get("preparation_version") == ETH3D_SLAM_PREPARATION_VERSION
         and (archive is None or preparation.get("source") == archive.record())
@@ -254,15 +248,47 @@ def _image_dimensions(images: Sequence[tuple[Path, float]]) -> tuple[int, int]:
     return width, height
 
 
+def ground_truth_poses_at_timestamps(
+    image_times, timestamps, translations, quaternions, *, max_interpolation_timespan=1 / 75
+):
+    """Interpolate world-from-camera GT using the official ETH3D support rule.
+
+    ETH3D excludes extrapolation and brackets wider than its capture-specific limit.
+    In particular, a nearby endpoint does not make a tracking gap valid GT.
+    """
+    poses = []
+    for timestamp in image_times:
+        after = int(np.searchsorted(timestamps, timestamp, side="right"))
+        if (
+            after == 0
+            or after == len(timestamps)
+            or timestamps[after] - timestamps[after - 1] > max_interpolation_timespan
+        ):
+            poses.append(None)
+            continue
+        before = after - 1
+        fraction = (timestamp - timestamps[before]) / (timestamps[after] - timestamps[before])
+        endpoints = [
+            pycolmap.Rigid3d(pycolmap.Rotation3d(quaternions[index]), translations[index]) for index in (before, after)
+        ]
+        poses.append(pycolmap.Rigid3d.interpolate(*endpoints, fraction))
+    return poses
+
+
 def _convert_scene(source: Path, destination: Path, *, gt_available: bool = True) -> list[int]:
     calibration = _read_calibration(source / "calibration.txt")
     images = _source_images(source)
     width, height = _image_dimensions(images)
-    assigned_poses = None
-    translations = quaternions = None
+    poses = [None] * len(images)
     if gt_available:
         timestamps, translations, quaternions = _read_groundtruth(source / "groundtruth.txt")
-        assigned_poses = assign_images_to_timestamp_ids((timestamp for _, timestamp in images), timestamps)
+        poses = ground_truth_poses_at_timestamps(
+            (timestamp for _, timestamp in images),
+            timestamps,
+            translations,
+            quaternions,
+            max_interpolation_timespan=1 / 11.25 if source.name.startswith("sfm_") else 1 / 75,
+        )
 
     reconstruction = pycolmap.Reconstruction()
     camera = pycolmap.Camera(model="PINHOLE", width=width, height=height, params=calibration, camera_id=0)
@@ -275,14 +301,14 @@ def _convert_scene(source: Path, destination: Path, *, gt_available: bool = True
     unposed_names = []
     for image_id, (source_image, _) in enumerate(images):
         image = pycolmap.Image(camera_id=0, name=f"cam0/{source_image.name}", image_id=image_id)
-        if assigned_poses is None:
-            camera_from_world = pycolmap.Rigid3d()
-            unposed_names.append(f"cam0/{source_image.name}")
+        pose = poses[image_id]
+        if pose is None:
+            unposed_names.append(image.name)
+            # COLMAP omits unregistered images when writing; the sidecar removes
+            # this placeholder pose on load while preserving the input image.
+            reconstruction.add_image_with_trivial_frame(image, pycolmap.Rigid3d())
         else:
-            pose_id = assigned_poses[image_id]
-            rotation = pycolmap.Rotation3d(quaternions[pose_id])
-            camera_from_world = pycolmap.Rigid3d(rotation=rotation, translation=translations[pose_id]).inverse()
-        reconstruction.add_image_with_trivial_frame(image, camera_from_world)
+            reconstruction.add_image_with_trivial_frame(image, pose.inverse())
         shutil.copy2(source_image, images_dir / source_image.name)
 
     reconstruction.write_binary(rec_dir)

vidmap/utils/trajectory.py

@@ -122,6 +122,8 @@ def _add_target_image(reconstruction, target_image, pose) -> None:
 
 
 def _exact_keyframe_remap(target, source, target_timeline, source_timeline) -> PoseRemap | None:
+    unavailable = {image.name for image in target.images.values() if not image.has_pose}
+    source_timeline = tuple(item for item in source_timeline if source.images[item.image_id].name not in unavailable)
     target_matches = match_timestamps(
         (item.timestamp for item in source_timeline),
         (item.timestamp for item in target_timeline),
@@ -148,12 +150,14 @@ def remap_poses_to_timeline(
     source,
     *,
     prefer_exact_keyframes: bool = False,
+    interpolate_missing_poses: bool = True,
 ) -> PoseRemap:
     """Map source poses to target image IDs by timestamp.
 
     Exact source keyframes can be retained without interpolation when every
     source timestamp exists on the target timeline. Otherwise, source poses are
     interpolated at every target timestamp inside the source time range.
+    Disable filling explicitly unposed source images when the source is GT.
     """
     target_timeline = _posed_timeline(target)
     source_timeline = _posed_timeline(source)
@@ -177,10 +181,17 @@ def remap_poses_to_timeline(
         (item.timestamp for item in source_timeline),
     )
     remapped = _empty_pose_reconstruction(target)
+    unavailable = (
+        {image.name for image in source.images.values() if not image.has_pose}
+        if not interpolate_missing_poses
+        else set()
+    )
     for target_item, match in zip(target_timeline, matches, strict=True):
         if match.fraction is None:
             continue
         target_image = target.images[target_item.image_id]
+        if target_image.name in unavailable:
+            continue
         if match.is_exact:
             source_item = source_timeline[match.before]
             pose = source.images[source_item.image_id].cam_from_world()
@@ -220,10 +231,10 @@ def align_reconstruction_to_reference_sequence(
     colmap_bin: str = "colmap",
 ) -> tuple[pycolmap.Reconstruction, pycolmap.Sim3d]:
     """Align a reconstruction to the posed timeline in a reference model."""
-    posed_image_ids = [image_id for image_id, image in reference.images.items() if image.has_pose]
     mapped_reference = remap_poses_to_timeline(
         reconstruction,
-        _select_subreconstruction(posed_image_ids, reference),
+        reference,
+        interpolate_missing_poses=False,
     ).reconstruction
 
     with tempfile.TemporaryDirectory() as tmpdir:
@@ -261,7 +272,9 @@ def align_reconstruction_to_reference_sequence(
             used_translation_fallback = True
             shutil.copytree(source_dir, aligned_dir, dirs_exist_ok=True)
 
-            source_centers = [image.projection_center() for image in reconstruction.images.values() if image.has_pose]
+            source_centers = [
+                reconstruction.images[image_id].projection_center() for image_id in mapped_reference.images
+            ]
             reference_centers = [
                 image.projection_center() for image in mapped_reference.images.values() if image.has_pose
             ]