From 80fcbef20874549a7a781509ba8822f911d6fd7a Mon Sep 17 00:00:00 2001 From: Zador-Pataki Date: Fri, 30 Oct 2026 20:37:23 +0100 Subject: [PATCH] Fix ETH3D ground-truth preparation and evaluation --- README.md | 2 +- vidmap/benchmark/trajectory.py | 3 ++ vidmap/benchmarking.py | 2 +- vidmap/datasets/base.py | 4 +- vidmap/datasets/eth3d_slam.py | 2 + vidmap/datasets/prepare/eth3d_slam.py | 60 +++++++++++++++++++-------- vidmap/utils/trajectory.py | 19 +++++++-- 7 files changed, 69 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 5bdb8ff..bbaa535 100644 --- a/README.md +++ b/README.md @@ -527,7 +527,7 @@ Pose AUC, averaged equally across sequences (%, higher is better): | Dataset | 5 cm | 10 cm | 50 cm | 1 m | 10 m | | :--- | ---: | ---: | ---: | ---: | ---: | | EuRoC (11 sequences) | 28.86 | 60.02 | 91.96 | 95.98 | 99.60 | -| ETH3D-SLAM (55 sequences) | 63.96 | 74.59 | 89.19 | 93.25 | 98.91 | +| ETH3D-SLAM (55 sequences) | 78.48 | 85.74 | 94.39 | 96.61 | 99.52 | diff --git a/vidmap/benchmark/trajectory.py b/vidmap/benchmark/trajectory.py index 64e40f2..d7f2477 100644 --- a/vidmap/benchmark/trajectory.py +++ b/vidmap/benchmark/trajectory.py @@ -409,6 +409,7 @@ def evaluate( remap = remap_poses_to_timeline( ground_truth_reconstruction, estimated_reconstruction, + # Full-GT evaluation must retain missing estimates as failures. prefer_exact_keyframes=True, ) data = { @@ -423,6 +424,8 @@ def evaluate( ground_truth = ground_truth_reconstruction if remap.target_subset is not None and not self.options.eval_on_full_gt_timeline: ground_truth = remap.target_subset + if ground_truth.num_reg_images() == 0: + raise ValueError("No ground-truth-supported poses available for evaluation") data |= self.evaluate_aligned_reconstructions(remap.reconstruction, ground_truth) data["results"]["scale"] = self.options.gt_scale diff --git a/vidmap/benchmarking.py b/vidmap/benchmarking.py index 2a8a113..6f36096 100644 --- a/vidmap/benchmarking.py +++ b/vidmap/benchmarking.py @@ -199,7 +199,7 @@ def run( align_reconstruction_for_evaluation(estimated_reconstruction, scene_parser.rec) estimated_trajectory = reconstruction_to_evo_trajectory(estimated_reconstruction) aligned_ground_truth_reconstruction = remap_poses_to_timeline( - estimated_reconstruction, scene_parser.rec + estimated_reconstruction, scene_parser.rec, interpolate_missing_poses=False ).reconstruction ground_truth_trajectory = reconstruction_to_evo_trajectory(aligned_ground_truth_reconstruction) absolute_trajectory_error = compute_ate_statistics(ground_truth_trajectory, estimated_trajectory) diff --git a/vidmap/datasets/base.py b/vidmap/datasets/base.py index d4f4c64..8b0040a 100644 --- a/vidmap/datasets/base.py +++ b/vidmap/datasets/base.py @@ -271,7 +271,9 @@ def __init__(self, layout: DatasetLayout, scene: str) -> None: if unknown: raise ValueError(f"Canonical unposed-image sidecar references unknown images: {sorted(unknown)}") unposed_names.update(sidecar_names) - for frame_id in sorted({images_by_name[name].frame_id for name in unposed_names}): + for frame_id in sorted( + {images_by_name[name].frame_id for name in unposed_names if images_by_name[name].has_pose} + ): self.rec.deregister_frame(frame_id) def camera(self, imid: int) -> pycolmap.Camera: diff --git a/vidmap/datasets/eth3d_slam.py b/vidmap/datasets/eth3d_slam.py index fd46cbe..cb0f52e 100644 --- a/vidmap/datasets/eth3d_slam.py +++ b/vidmap/datasets/eth3d_slam.py @@ -2,6 +2,8 @@ from vidmap.datasets.video import VideoDatasetManifest, VideoSequence +ETH3D_SLAM_PREPARATION_VERSION = 3 + ETH3D_SLAM_SCENES = ( "cables_1", "cables_2", diff --git a/vidmap/datasets/prepare/eth3d_slam.py b/vidmap/datasets/prepare/eth3d_slam.py index 22f145a..119d61e 100644 --- a/vidmap/datasets/prepare/eth3d_slam.py +++ b/vidmap/datasets/prepare/eth3d_slam.py @@ -17,16 +17,14 @@ from PIL import Image from vidmap.datasets.base import CANONICAL_UNPOSED_IMAGES_FILENAME, DatasetLayout -from vidmap.datasets.eth3d_slam import ETH3D_SLAM_MANIFEST +from vidmap.datasets.eth3d_slam import ETH3D_SLAM_MANIFEST, ETH3D_SLAM_PREPARATION_VERSION from vidmap.datasets.layouts import get_dataset_layout from vidmap.datasets.prepare import filesystem as prep_fs -from vidmap.datasets.prepare.timestamp_matching import assign_images_to_timestamp_ids from vidmap.datasets.prepare.video_common import resumable_http_download, safe_extract_zip from vidmap.datasets.resources import dataset_asset _URL_TEMPLATE = "https://www.eth3d.net/data/slam/datasets/{scene}_mono.zip" _SOURCE_RECORD = ".source_archive.json" -ETH3D_SLAM_PREPARATION_VERSION = 2 ETH3D_SLAM_LAYOUT = get_dataset_layout("eth3d_slam") @@ -134,16 +132,12 @@ def _scene_is_complete(layout: DatasetLayout, scene: str) -> bool: archive = _source_archive(scene) except ValueError: archive = None - expected_unposed = None - if archive is not None: - expected_unposed = set() if archive.gt_available else names return ( bool(reconstruction.cameras) and bool(image_ids) and testsets == {"0": image_ids} and unposed is not None and set(unposed) <= names - and (expected_unposed is None or set(unposed) == expected_unposed) and isinstance(preparation, Mapping) and preparation.get("preparation_version") == ETH3D_SLAM_PREPARATION_VERSION and (archive is None or preparation.get("source") == archive.record()) @@ -254,15 +248,47 @@ def _image_dimensions(images: Sequence[tuple[Path, float]]) -> tuple[int, int]: return width, height +def ground_truth_poses_at_timestamps( + image_times, timestamps, translations, quaternions, *, max_interpolation_timespan=1 / 75 +): + """Interpolate world-from-camera GT using the official ETH3D support rule. + + ETH3D excludes extrapolation and brackets wider than its capture-specific limit. + In particular, a nearby endpoint does not make a tracking gap valid GT. + """ + poses = [] + for timestamp in image_times: + after = int(np.searchsorted(timestamps, timestamp, side="right")) + if ( + after == 0 + or after == len(timestamps) + or timestamps[after] - timestamps[after - 1] > max_interpolation_timespan + ): + poses.append(None) + continue + before = after - 1 + fraction = (timestamp - timestamps[before]) / (timestamps[after] - timestamps[before]) + endpoints = [ + pycolmap.Rigid3d(pycolmap.Rotation3d(quaternions[index]), translations[index]) for index in (before, after) + ] + poses.append(pycolmap.Rigid3d.interpolate(*endpoints, fraction)) + return poses + + def _convert_scene(source: Path, destination: Path, *, gt_available: bool = True) -> list[int]: calibration = _read_calibration(source / "calibration.txt") images = _source_images(source) width, height = _image_dimensions(images) - assigned_poses = None - translations = quaternions = None + poses = [None] * len(images) if gt_available: timestamps, translations, quaternions = _read_groundtruth(source / "groundtruth.txt") - assigned_poses = assign_images_to_timestamp_ids((timestamp for _, timestamp in images), timestamps) + poses = ground_truth_poses_at_timestamps( + (timestamp for _, timestamp in images), + timestamps, + translations, + quaternions, + max_interpolation_timespan=1 / 11.25 if source.name.startswith("sfm_") else 1 / 75, + ) reconstruction = pycolmap.Reconstruction() camera = pycolmap.Camera(model="PINHOLE", width=width, height=height, params=calibration, camera_id=0) @@ -275,14 +301,14 @@ def _convert_scene(source: Path, destination: Path, *, gt_available: bool = True unposed_names = [] for image_id, (source_image, _) in enumerate(images): image = pycolmap.Image(camera_id=0, name=f"cam0/{source_image.name}", image_id=image_id) - if assigned_poses is None: - camera_from_world = pycolmap.Rigid3d() - unposed_names.append(f"cam0/{source_image.name}") + pose = poses[image_id] + if pose is None: + unposed_names.append(image.name) + # COLMAP omits unregistered images when writing; the sidecar removes + # this placeholder pose on load while preserving the input image. + reconstruction.add_image_with_trivial_frame(image, pycolmap.Rigid3d()) else: - pose_id = assigned_poses[image_id] - rotation = pycolmap.Rotation3d(quaternions[pose_id]) - camera_from_world = pycolmap.Rigid3d(rotation=rotation, translation=translations[pose_id]).inverse() - reconstruction.add_image_with_trivial_frame(image, camera_from_world) + reconstruction.add_image_with_trivial_frame(image, pose.inverse()) shutil.copy2(source_image, images_dir / source_image.name) reconstruction.write_binary(rec_dir) diff --git a/vidmap/utils/trajectory.py b/vidmap/utils/trajectory.py index ec2ab62..d3f409c 100644 --- a/vidmap/utils/trajectory.py +++ b/vidmap/utils/trajectory.py @@ -122,6 +122,8 @@ def _add_target_image(reconstruction, target_image, pose) -> None: def _exact_keyframe_remap(target, source, target_timeline, source_timeline) -> PoseRemap | None: + unavailable = {image.name for image in target.images.values() if not image.has_pose} + source_timeline = tuple(item for item in source_timeline if source.images[item.image_id].name not in unavailable) target_matches = match_timestamps( (item.timestamp for item in source_timeline), (item.timestamp for item in target_timeline), @@ -148,12 +150,14 @@ def remap_poses_to_timeline( source, *, prefer_exact_keyframes: bool = False, + interpolate_missing_poses: bool = True, ) -> PoseRemap: """Map source poses to target image IDs by timestamp. Exact source keyframes can be retained without interpolation when every source timestamp exists on the target timeline. Otherwise, source poses are interpolated at every target timestamp inside the source time range. + Disable filling explicitly unposed source images when the source is GT. """ target_timeline = _posed_timeline(target) source_timeline = _posed_timeline(source) @@ -177,10 +181,17 @@ def remap_poses_to_timeline( (item.timestamp for item in source_timeline), ) remapped = _empty_pose_reconstruction(target) + unavailable = ( + {image.name for image in source.images.values() if not image.has_pose} + if not interpolate_missing_poses + else set() + ) for target_item, match in zip(target_timeline, matches, strict=True): if match.fraction is None: continue target_image = target.images[target_item.image_id] + if target_image.name in unavailable: + continue if match.is_exact: source_item = source_timeline[match.before] pose = source.images[source_item.image_id].cam_from_world() @@ -220,10 +231,10 @@ def align_reconstruction_to_reference_sequence( colmap_bin: str = "colmap", ) -> tuple[pycolmap.Reconstruction, pycolmap.Sim3d]: """Align a reconstruction to the posed timeline in a reference model.""" - posed_image_ids = [image_id for image_id, image in reference.images.items() if image.has_pose] mapped_reference = remap_poses_to_timeline( reconstruction, - _select_subreconstruction(posed_image_ids, reference), + reference, + interpolate_missing_poses=False, ).reconstruction with tempfile.TemporaryDirectory() as tmpdir: @@ -261,7 +272,9 @@ def align_reconstruction_to_reference_sequence( used_translation_fallback = True shutil.copytree(source_dir, aligned_dir, dirs_exist_ok=True) - source_centers = [image.projection_center() for image in reconstruction.images.values() if image.has_pose] + source_centers = [ + reconstruction.images[image_id].projection_center() for image_id in mapped_reference.images + ] reference_centers = [ image.projection_center() for image in mapped_reference.images.values() if image.has_pose ]