From 707b6437220b4d9393ac5bf722a21c50f2409ece Mon Sep 17 00:00:00 2001 From: Zador-Pataki Date: Sun, 25 Oct 2026 18:06:28 +0100 Subject: [PATCH] Support calibrated camera rigs --- README.md | 20 +- docs/assets/rig-example.svg | 210 ++++++++++ docs/cameras.md | 71 ++++ docs/files.md | 95 +++++ extensions/colmap/CMakeLists.txt | 2 +- .../src/bindings/bundle_adjustment_problem.cc | 9 +- .../bindings/global_positioning_problem.cc | 181 ++++++--- .../colmap/src/bindings/mapping_sidecars.cc | 16 +- extensions/colmap/src/bindings/tracks.cc | 66 ++++ extensions/colmap/src/stages/depth_prior.h | 24 +- .../colmap/src/stages/image_pair_inliers.cc | 16 + extensions/colmap/src/stages/metric_depth.h | 24 +- extensions/colmap/src/stages/rig_geometry.h | 42 ++ .../colmap/src/stages/track_establishment.cc | 54 ++- .../tests/python/test_global_positioning.py | 28 +- .../tests/python/test_rotation_averaging.py | 34 +- vidmap/calibration.py | 57 +++ vidmap/configs/frontend/calib/rig.yaml | 10 + vidmap/configuration/config.py | 8 + vidmap/configuration/dump.py | 6 + vidmap/datasets/capture.py | 363 ++++++++++++++++++ vidmap/datasets/frame_names.py | 30 +- vidmap/datasets/local.py | 124 +----- vidmap/datasets/prepare/filesystem.py | 39 ++ vidmap/datasets/prepare/rig_capture.py | 127 ++++++ vidmap/frontend/cache/validation.py | 2 +- vidmap/frontend/cli.py | 2 - vidmap/frontend/colmap_database.py | 158 +++----- vidmap/frontend/depth.py | 33 +- vidmap/frontend/depth_loading.py | 24 +- vidmap/frontend/identity.py | 29 +- vidmap/frontend/image_dataset.py | 84 ++-- vidmap/frontend/initial_reconstruction.py | 20 - vidmap/frontend/keyframes/cache.py | 168 +++----- vidmap/frontend/keyframes/processing.py | 175 ++++++--- vidmap/frontend/keyframes/selector.py | 8 +- vidmap/frontend/loop_closure/cache.py | 41 +- .../frontend/loop_closure/extended_matches.py | 83 ++-- vidmap/frontend/loop_closure/matching.py | 82 ++-- vidmap/frontend/loop_closure/retrieval.py | 111 +++--- vidmap/frontend/options/__init__.py | 9 +- vidmap/frontend/options/matching.py | 6 +- vidmap/frontend/options/preparation.py | 10 + vidmap/frontend/options/rig.py | 63 +++ vidmap/frontend/paths.py | 10 +- vidmap/frontend/pipeline.py | 81 ++-- vidmap/frontend/preparation/camera_priors.py | 57 ++- vidmap/frontend/preparation/cameras.py | 122 ++++++ .../preparation/geometric_verification.py | 43 +-- vidmap/frontend/preparation/mapper_inputs.py | 67 ++-- vidmap/frontend/rig.py | 54 +++ vidmap/frontend/runner.py | 133 +++++-- vidmap/frontend/tracking/composition.py | 97 +++-- vidmap/frontend/tracking/propagation.py | 30 +- vidmap/frontend/tracking/sparse_tracks.py | 153 +++----- vidmap/frontend/tracking/transitive.py | 129 +++---- vidmap/mapper/focal_prior.py | 27 +- vidmap/mapper/inputs/__init__.py | 2 - vidmap/mapper/inputs/database.py | 16 +- vidmap/mapper/inputs/loader.py | 24 +- vidmap/mapper/inputs/snapshot.py | 111 +++--- vidmap/mapper/inputs/timeline.py | 75 ++++ vidmap/mapper/mapper.py | 31 +- .../stages/bundle_adjustment/adjuster.py | 10 +- .../stages/global_positioning/positioner.py | 85 ++-- .../stages/global_positioning/problem.py | 96 +++-- .../mapper/stages/relative_pose/estimator.py | 60 ++- vidmap/mapper/stages/rotation_averaging.py | 32 +- vidmap/reconstruction.py | 71 +--- vidmap/run.py | 16 +- vidmap/runtime_variants.py | 2 +- .../html/browser/browser_reconstruction.js | 77 +++- .../html/browser/browser_scene.js | 229 ++++++++--- .../html/browser/browser_worker.js | 146 ++++++- .../html/browser/vidmap-viewer.html.in | 29 +- vidmap/visualization/html/cli.py | 2 +- vidmap/visualization/html/embedded.py | 54 +-- vidmap/visualization/vidmap-viewer.html | 337 ++++++++++++---- 78 files changed, 3737 insertions(+), 1535 deletions(-) create mode 100644 docs/assets/rig-example.svg create mode 100644 docs/cameras.md create mode 100644 docs/files.md create mode 100644 extensions/colmap/src/stages/rig_geometry.h create mode 100644 vidmap/calibration.py create mode 100644 vidmap/configs/frontend/calib/rig.yaml create mode 100644 vidmap/datasets/capture.py create mode 100644 vidmap/datasets/prepare/rig_capture.py delete mode 100644 vidmap/frontend/initial_reconstruction.py create mode 100644 vidmap/frontend/options/rig.py create mode 100644 vidmap/frontend/preparation/cameras.py create mode 100644 vidmap/frontend/rig.py create mode 100644 vidmap/mapper/inputs/timeline.py diff --git a/README.md b/README.md index 371414f..9392045 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ estimate camera poses, camera intrinsics, and a sparse 3D map. - πŸ”§ [Setup](#setup) β€” Install dependencies and prepare the environment. - πŸš€ [Run VidMap](#execution) β€” Reconstruct a video or image sequence. +- πŸ“· [Camera rigs](#camera-rigs) β€” Configure your cameras. - πŸ‘€ [Visualization](#visualization) β€” Explore your reconstruction in the browser or Rerun. - πŸ› οΈ [Pipeline Configurations](#configuration) β€” Customize the frontend and mapping settings. - ⚑ [Speed up VidMap](#speed-up-vidmap) β€” Find settings to make extraction and mapping faster. @@ -46,7 +47,7 @@ git clone --recursive https://github.com/cvg/vidmap.git && cd vidmap Build and install [COLMAP](https://github.com/colmap/colmap) and its PyCOLMAP bindings [from source](https://colmap.github.io/install.html#build-from-source). -Last tested with [9e653da](https://github.com/colmap/colmap/commit/9e653da3baaaada926bda6c5286568413fb3e675). +Last tested with: [COLMAP `b1972bc2`](https://github.com/colmap/colmap/commit/b1972bc226f58e095c8b75bc214f4bb2d8ba57e4). For GPU mapper acceleration, build them against [Ceres 2.3 or newer](https://ceres-solver.readthedocs.io/latest/installation.html) with CUDA and cuDSS instead ([see how to enable](#gpu-acceleration)). @@ -103,6 +104,23 @@ Both mapping commands write the final COLMAP model under `$OUTPUT_DIR/rec`. Resolved `frontend_config.yaml` and `mapping_config.yaml` are written directly under `$OUTPUT_DIR`. +See [files and folders](docs/files.md) for image layouts, frame associations, +configuration files and run artifacts. + +### Camera rigs + +| Input | Uncalibrated mono | Calibrated mono | Uncalibrated rig | Calibrated rig | +|---|:---:|:---:|:---:|:---:| +| Pinhole | [βœ“](docs/cameras.md#monocular) | [βœ“](docs/cameras.md#calibrated-monocular) | βœ— | [βœ“](docs/cameras.md#camera-rigs) | +| Distorted perspective | βœ— | [βœ“](docs/cameras.md#calibrated-monocular)ΒΉ | βœ— | [βœ“](docs/cameras.md#camera-rigs)ΒΉ | +| Fisheye | βœ— | [βœ“](docs/cameras.md#calibrated-monocular)ΒΉ | βœ— | [βœ“](docs/cameras.md#camera-rigs)ΒΉ | + +Distorted perspective models: `SIMPLE_RADIAL`, `RADIAL`, `OPENCV`, `FULL_OPENCV`, +`SIMPLE_DIVISION`, `DIVISION`. See [COLMAP camera models](https://colmap.github.io/cameras.html) +for model definitions and parameter order. + +1. Undistorted to pinhole before mapping. Calibrated rigs may mix camera models. + ## Visualization ### Browser viewer diff --git a/docs/assets/rig-example.svg b/docs/assets/rig-example.svg new file mode 100644 index 0000000..57a70d8 --- /dev/null +++ b/docs/assets/rig-example.svg @@ -0,0 +1,210 @@ + + + + + + + + image/svg+xml + + + Matplotlib v3.10.3, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + 1 + + + 2 + + + 3 + + + + + + + + + diff --git a/docs/cameras.md b/docs/cameras.md new file mode 100644 index 0000000..aaa7005 --- /dev/null +++ b/docs/cameras.md @@ -0,0 +1,71 @@ +# Camera setup + +Set your input/output paths and choose the config names below (without `.yaml`): + +```bash +python -m vidmap.run --input_data /data/images --output /results/run \ + --frontend-conf FRONTEND_CONFIG --mapping-conf MAPPING_CONFIG +``` + +| Input | `FRONTEND_CONFIG` | `MAPPING_CONFIG` | +|---|---|---| +| Uncalibrated monocular | `uncalib/base` | `uncalib/base` | +| Calibrated examples below | Saved config name | `calib/base` | + +## Monocular + +The default configuration uses uncalibrated monocular input. + +### Calibrated monocular + +Use a one-camera rig. Save `vidmap/configs/frontend/my_camera.yaml`: + +```yaml +defaults: [calib/base] +rig: + cameras: + - camera_model_name: PINHOLE + camera_params: [500, 500, 420, 420] +``` + +Replace the calibration with yours. Put timestamp-named images (e.g. `1.000.png`) +in `/data/images/cam0/`. + +## Camera rigs + +Example four-camera rig: a stereo pair (0–1), an oblique camera (2), and a rear camera (3). + +![Four-camera rig viewing directions](assets/rig-example.svg) + +Use synchronized images in `cam0/` through `cam3/`; +see [input folders and frame association](files.md#input-images). Save `vidmap/configs/frontend/my_rig.yaml`: + +```yaml +defaults: [calib/rig] +rig: + cameras: + - camera_model_name: PINHOLE + camera_params: [500, 500, 420, 420] # fx, fy, cx, cy (pixels) + - camera_model_name: PINHOLE + camera_params: [500, 500, 420, 420] + cam_from_rig_rotation: [1, 0, 0, 0] + cam_from_rig_translation: [-0.1, 0, 0] + - camera_model_name: PINHOLE + camera_params: [500, 500, 420, 420] + cam_from_rig_rotation: [0.9238795325112867, 0, -0.3826834323650898, 0] + cam_from_rig_translation: [-0.1414214, 0, -0.1414214] + - camera_model_name: PINHOLE + camera_params: [500, 500, 420, 420] + cam_from_rig_rotation: [0, 0, 1, 0] + cam_from_rig_translation: [0.1, 0, -0.15] + match_same_timestamp: [[0, 1], [1, 2]] + flag_same_timestamp_as_sequential: [[0, 1]] +``` + +Camera 0 is the reference and the default keyframing camera. + +In this example: + +- **0–1:** Reliable stereo overlap: included in `match_same_timestamp` and `flag_same_timestamp_as_sequential` for sequential losses. +- **1–2:** Overlap with a larger viewpoint change: included only in `match_same_timestamp`, using loop-closure losses. +- **3:** No overlap: omitted from `match_same_timestamp`. diff --git a/docs/files.md b/docs/files.md new file mode 100644 index 0000000..5d1ce5a --- /dev/null +++ b/docs/files.md @@ -0,0 +1,95 @@ +# Files and folders + +## Input images + +JPG, JPEG and PNG are supported. + +| Input | `--input_data` points to | +|---|---| +| Uncalibrated monocular | The image folder itself, or an MP4/MOV video | +| Calibrated monocular | A directory containing `cam0/` | +| Physical rig | A directory containing `cam0/`, `cam1/`, … | + +Uncalibrated monocular images use natural filename order (`frame2.png` before +`frame10.png`). Rigs and calibrated monocular resolve numeric filenames +automatically: `000001.png`, `1.100.png` or `1.100-left.png`. Numbers are seconds; +every `camN/` folder must contain exactly one image per shared timestamp. + +### Frame association + +If filenames cannot establish associations, set `frames: frames.csv` inside your +frontend config’s `rig` section. For `--input_data /data/images`: + +```text +/data/images/ +β”œβ”€β”€ frames.csv +β”œβ”€β”€ cam0/ +β”‚ β”œβ”€β”€ a.png +β”‚ └── b.png +└── cam1/ + β”œβ”€β”€ left_a.png + └── left_b.png +``` + +`/data/images/frames.csv`: + +```csv +frame_id,timestamp_ns,camera_id,image_name +10,1000000000,0,cam0/a.png +10,1000000000,1,cam1/left_a.png +20,1100000000,0,cam0/b.png +20,1100000000,1,cam1/left_b.png +``` + +## Configuration files + +| Files | Select with | +|---|---| +| `vidmap/configs/frontend/.yaml` | `--frontend-conf ` | +| `vidmap/configs/mapping/.yaml` | `--mapping-conf ` | + +Use names without `.yaml`, e.g. `uncalib/base`. Inherit presets with +`defaults: [calib/rig]`; see [camera examples](cameras.md). + +## Run outputs + +Local runs write under `--output`: + +```text +/results/run/ +β”œβ”€β”€ frontend_config.yaml # Resolved frontend settings +β”œβ”€β”€ mapping_config.yaml # Resolved mapping settings +β”œβ”€β”€ mapper_inputs/ # Reusable frontend results (below) +β”œβ”€β”€ database_complete.db # Mapping's working database copy +└── rec/ # Final COLMAP reconstruction +``` + +Optional outputs: + +| Path | Contents | +|---|---| +| `video_frames//` | Decoded video images | +| `prepared_capture/images/` | Undistorted rig images | +| `full_depth_maps.h5` | Full depth maps retained with `--cache-depth-maps` | +| `vidmap-viewer-embedded.html` | Standalone viewer, after [export](../README.md#browser-viewer) | + +Original image folders remain at their input paths. Use a new output directory +when changing sources or calibration. + +### Mapper inputs + +Keep this directory together to rerun mapping without extraction: + +| File in `mapper_inputs/` | Contents | +|---|---| +| `database_complete.db` | Cameras, rigs, frames, keypoints and matches | +| `depth_maps.h5` | Depth priors and validity | +| `lc_masks.json` | Sequential / loop-closure match labels | +| `manifest.json` | Input identity, frame order and file checksums | +| `local_input.json` | Source image paths for local runs | +| `geocalib_per_image.h5`, `geocalib_batch.h5` | GeoCalib priors, when generated | +| `vgc_filtered_pairs.json` | Pairs excluded from calibration, when generated | + +[Rerun mapping](../README.md#execution) with `--mapper-inputs` and the original +frontend config. External mapper inputs are neither modified nor copied into +the output folder. diff --git a/extensions/colmap/CMakeLists.txt b/extensions/colmap/CMakeLists.txt index 6ddc1c3..c1291bd 100644 --- a/extensions/colmap/CMakeLists.txt +++ b/extensions/colmap/CMakeLists.txt @@ -17,7 +17,7 @@ find_package(pybind11 3.1.0 CONFIG REQUIRED) set(VIDMAP_COLMAP_VERSION "4.3.0") set( VIDMAP_COLMAP_REVISION - "9e653da3baaaada926bda6c5286568413fb3e675" + "b1972bc226f58e095c8b75bc214f4bb2d8ba57e4" CACHE STRING "Upstream COLMAP revision used by vidmap_native" ) diff --git a/extensions/colmap/src/bindings/bundle_adjustment_problem.cc b/extensions/colmap/src/bindings/bundle_adjustment_problem.cc index cf048ba..c0e3039 100644 --- a/extensions/colmap/src/bindings/bundle_adjustment_problem.cc +++ b/extensions/colmap/src/bindings/bundle_adjustment_problem.cc @@ -5,6 +5,7 @@ #include "stages/depth_prior.h" #include "stages/intrinsics_prior.h" +#include "stages/rig_geometry.h" #include #include #include @@ -41,6 +42,7 @@ PYBIND11_MODULE(bundle_adjustment, m) { using Losses = std::vector>; auto losses = std::make_unique(); auto& image = reconstruction.Image(image_id); + const auto sensor_from_rig = vidmap::FixedSensorFromRig(image); double* pose = image.FramePtr()->RigFromWorld().params.data(); if (fix_pose || problem.HasParameterBlock(pose)) { losses->reserve(count); @@ -48,17 +50,14 @@ PYBIND11_MODULE(bundle_adjustment, m) { if (weights[i] == 0.0) continue; double* point = reconstruction.Point3D(point_ids[i]).xyz.data(); if (!problem.HasParameterBlock(point)) continue; - if (!image.IsRefInFrame()) { - throw std::invalid_argument( - "VidMap depth constraints require a reference camera"); - } losses->push_back(colmap::CreateCeresLossFunction( robust_mask[i] ? loss_type : colmap::CeresLossFunctionType::TRIVIAL, robust_scales[i], weights[i])); problem.AddResidualBlock( - vidmap::LogScaledDepthErrorCostFunctor::Create(depths[i]), + vidmap::LogScaledDepthErrorCostFunctor::Create( + depths[i], sensor_from_rig), losses->back().get(), pose, point, diff --git a/extensions/colmap/src/bindings/global_positioning_problem.cc b/extensions/colmap/src/bindings/global_positioning_problem.cc index 9d044d1..341b3d2 100644 --- a/extensions/colmap/src/bindings/global_positioning_problem.cc +++ b/extensions/colmap/src/bindings/global_positioning_problem.cc @@ -4,12 +4,15 @@ #include "colmap/scene/camera.h" #include "colmap/scene/reconstruction.h" -#include +#include +#include #include +#include #include #include #include "stages/metric_depth.h" +#include "stages/rig_geometry.h" #include "stages/weighted_motion_averaging.h" #include "vidmap_native/mapping_sidecars.h" #include @@ -80,13 +83,13 @@ struct ObservationSelection { std::vector observations; }; using TrackMap = std::map; -using Centers = std::unordered_map; +using Centers = std::unordered_map; Centers CenterPointers(const py::dict& arrays) { Centers centers; for (const auto& item : arrays) { auto array = item.second.cast>(); - centers.emplace(item.first.cast(), array.mutable_data()); + centers.emplace(item.first.cast(), array.mutable_data()); } return centers; } @@ -101,31 +104,32 @@ std::map PrepareObservations( const vidmap::MappingSidecars& sidecars, size_t minimum_track_length, bool include_loop, - const std::vector& image_order, + const std::vector& frame_order, size_t support_count) { std::map selected; - std::unordered_map timeline; - for (size_t i = 0; i < image_order.size(); ++i) - timeline.emplace(image_order[i], i); + std::unordered_map timeline; + for (size_t i = 0; i < frame_order.size(); ++i) + timeline.emplace(frame_order[i], i); for (const auto& [point_id, point] : reconstruction.Points3D()) { const auto& regular = point.track.Elements(); if (regular.size() < minimum_track_length) continue; std::vector support(regular.size(), false); if (support_count) { - std::vector order(regular.size()); - std::iota(order.begin(), order.end(), 0); - const size_t count = std::min(order.size(), support_count); - std::partial_sort( - order.begin(), - order.begin() + count, - order.end(), - [&](auto a, auto b) { - return std::make_pair(timeline.at(regular[a].image_id), - regular[a].point2D_idx) < - std::make_pair(timeline.at(regular[b].image_id), - regular[b].point2D_idx); - }); - for (size_t i = 0; i < count; ++i) support[order[i]] = true; + std::set support_ranks; + for (const auto& element : regular) { + const auto rank = + timeline.at(reconstruction.Image(element.image_id).FrameId()); + if (support_ranks.size() < support_count || + rank < *support_ranks.rbegin()) { + support_ranks.insert(rank); + if (support_ranks.size() > support_count) + support_ranks.erase(std::prev(support_ranks.end())); + } + } + for (size_t i = 0; i < regular.size(); ++i) + support[i] = timeline.at( + reconstruction.Image(regular[i].image_id).FrameId()) <= + *support_ranks.rbegin(); } auto append = [&](const colmap::TrackElement& element, const colmap::TrackElement& anchor, @@ -135,11 +139,13 @@ std::map PrepareObservations( selected[{loop, image.CameraPtr()->HasPriorFocalLength(), early}] .observations.push_back({point_id, element, anchor, loop, early}); }; - std::unordered_set regular_images; + std::set> regular_features; for (size_t row = 0; row < regular.size(); ++row) { - if (!regular_images.insert(regular[row].image_id).second) + if (!regular_features + .emplace(regular[row].image_id, regular[row].point2D_idx) + .second) throw std::invalid_argument( - "A regular track must have at most one observation per image"); + "A regular track must not repeat the same image feature"); append(regular[row], {}, false, support[row]); } const auto extra = sidecars.tracks.find(point_id); @@ -177,6 +183,8 @@ PYBIND11_MAKE_OPAQUE(TrackMap); PYBIND11_MODULE(global_positioning, m) { py::module_::import("pyceres"); + py::module_::import("pycolmap"); + m.def("validate_fixed_rigs", &vidmap::ValidateFixedRigScene); py::classh(m, "MutableLoss") .def(py::init([](ceres::LossFunction* loss) { return std::make_unique( @@ -218,39 +226,78 @@ PYBIND11_MODULE(global_positioning, m) { for (auto& [id, track] : tracks) std::swap(reconstruction.Point3D(id).track, track); }); - m.def("collect_default_observations", - [](ceres::Problem& problem, - colmap::Reconstruction& reconstruction, - const py::dict& centers, - ObservationSelection& selected) { - std::unordered_map points; - for (const auto& [id, point] : reconstruction.Points3D()) - points.emplace(point.xyz.data(), id); - std::unordered_map images; - for (const auto& [id, pointer] : CenterPointers(centers)) - images.emplace(pointer, id); - std::map, Observation*> - observations; - for (auto& obs : selected.observations) - observations.emplace(std::make_pair(obs.point_id, obs.ImageId()), - &obs); - std::vector residuals; - problem.GetResidualBlocks(&residuals); - for (auto residual : residuals) { - std::vector blocks; - problem.GetParameterBlocksForResidualBlock(residual, &blocks); - auto& obs = - *observations.at({points.at(blocks[1]), images.at(blocks[0])}); - obs.center = blocks[0]; - obs.point = blocks[1]; - obs.scale = blocks[2]; + m.def( + "collect_default_observations", + [](colmap::GlobalPositioner& positioner, + colmap::Reconstruction& reconstruction, + ObservationSelection& selected, + std::optional stddev) { + auto& problem = positioner.Problem(); + // Pinned COLMAP allocates one contiguous scale vector in native + // point/track traversal order. Ceres residual order is unspecified. + // Validate this layout before using it to identify repeated views of + // the same point in the same rig frame. + std::vector residuals; + problem.GetResidualBlocks(&residuals); + std::vector> blocks(residuals.size()); + uintptr_t first = std::numeric_limits::max(); + for (size_t i = 0; i < residuals.size(); ++i) { + problem.GetParameterBlocksForResidualBlock(residuals[i], &blocks[i]); + THROW_CHECK_EQ(blocks[i].size(), 3); + first = std::min(first, reinterpret_cast(blocks[i][2])); + } + std::vector*> ordered(residuals.size()); + for (const auto& block : blocks) { + const auto offset = reinterpret_cast(block[2]) - first; + THROW_CHECK_EQ(offset % sizeof(double), 0); + THROW_CHECK_LT(offset / sizeof(double), ordered.size()); + THROW_CHECK(ordered[offset / sizeof(double)] == nullptr); + ordered[offset / sizeof(double)] = █ + } + using Key = + std::tuple; + std::map observations; + for (auto& obs : selected.observations) + observations.emplace(Key{obs.point_id, obs.ImageId(), obs.Index()}, + &obs); + size_t index = 0; + for (auto& [point_id, point] : reconstruction.Points3D()) { + for (const auto& element : point.track.Elements()) { + const auto& image = reconstruction.Image(element.image_id); + if (!image.HasPose()) continue; + const auto& camera = *image.CameraPtr(); + const auto& pixel = image.Point2D(element.point2D_idx).xy; + std::optional ray; + if (stddev) { + const auto value = camera.CamRayFromImgWithJac(pixel); + if (!value) continue; + const Eigen::Matrix2d gram = + value->jacobian.transpose() * value->jacobian; + const double variance = 2.0 / gram.inverse().trace(); + if (!(variance > 0.0) || !std::isfinite(variance)) continue; + ray = value->ray; + } else { + ray = camera.CamRayFromImg(pixel); + } + if (!ray) continue; + THROW_CHECK_LT(index, ordered.size()); + const auto& block = *ordered[index++]; + auto& obs = *observations.at( + Key{point_id, element.image_id, element.point2D_idx}); + obs.point = reconstruction.Point3D(point_id).xyz.data(); + obs.center = positioner.FrameCenterParameterBlock(image.FrameId()); + obs.scale = block[2]; + THROW_CHECK_EQ(block[image.IsRefInFrame() ? 0 : 1], obs.center); + THROW_CHECK_EQ(block[image.IsRefInFrame() ? 1 : 0], obs.point); } - auto& obs = selected.observations; - obs.erase(std::remove_if(obs.begin(), - obs.end(), - [](const auto& o) { return !o.scale; }), - obs.end()); - }); + } + THROW_CHECK_EQ(index, ordered.size()); + auto& obs = selected.observations; + obs.erase( + std::remove_if( + obs.begin(), obs.end(), [](const auto& o) { return !o.scale; }), + obs.end()); + }); m.def("initialize_points", [](ceres::Problem& problem, colmap::Reconstruction& reconstruction, @@ -369,11 +416,13 @@ PYBIND11_MODULE(global_positioning, m) { ray = camera.CamRayFromImg(pixel); } if (!ray) continue; - double* center_data = center_pointers.at(image_id); + double* center_data = center_pointers.at(image.FrameId()); auto& point = reconstruction.Point3D(point_id).xyz; const Eigen::Vector3d direction = rotation.transpose() * *ray; - const Eigen::Vector3d delta = + Eigen::Vector3d delta = point - Eigen::Map(center_data); + const auto offset = vidmap::RigTranslationInWorld(image); + delta += offset; double* scale = scales.mutable_data(count); *scale = initialize_from_geometry @@ -385,11 +434,13 @@ PYBIND11_MODULE(global_positioning, m) { auto* cost = covariance ? colmap::CovarianceWeightedCostFunctor< - colmap::BATAPairwiseDirectionCostFunctor>:: - Create(*covariance, direction) - : colmap::BATAPairwiseDirectionCostFunctor::Create(direction); + colmap:: + RigBATAPairwiseDirectionConstantRigCostFunctor>:: + Create(*covariance, direction, offset) + : colmap::RigBATAPairwiseDirectionConstantRigCostFunctor:: + Create(direction, offset); problem.AddResidualBlock( - cost, loss.get(), center_data, point.data(), scale); + cost, loss.get(), point.data(), center_data, scale); if (!ordering->IsMember(point.data())) ordering->AddElementToGroup(point.data(), 1); problem.SetParameterLowerBound(scale, 0, 1e-5); @@ -458,7 +509,9 @@ PYBIND11_MODULE(global_positioning, m) { use_log_scales, residual_type, zero_residual_behind, - log_linear_threshold); + log_linear_threshold, + vidmap::RigTranslationInWorld( + reconstruction.Image(obs.ImageId()))); problem.AddResidualBlock(cost, loss.get(), obs.center, @@ -518,7 +571,7 @@ PYBIND11_MODULE(global_positioning, m) { b{anchor_image, obs.anchor.point2D_idx}; auto& edge = edges[std::minmax(obs.ImageId(), anchor_image)]; edge.matches.insert(std::minmax(a, b)); - const double* parameters[] = {obs.center, obs.point, obs.scale}; + const double* parameters[] = {obs.point, obs.center, obs.scale}; Eigen::Vector3d residual; // Evaluate the cost directly: Problem::Evaluate is unavailable in // callbacks. diff --git a/extensions/colmap/src/bindings/mapping_sidecars.cc b/extensions/colmap/src/bindings/mapping_sidecars.cc index 3e25764..ac01b51 100644 --- a/extensions/colmap/src/bindings/mapping_sidecars.cc +++ b/extensions/colmap/src/bindings/mapping_sidecars.cc @@ -1,6 +1,7 @@ #include "vidmap_native/mapping_sidecars.h" #include "bindings.h" +#include "stages/rig_geometry.h" #include #include #include @@ -84,14 +85,17 @@ void BindMappingSidecars(py::module_& m) { [](const colmap::Reconstruction& reconstruction, const std::vector& image_ids, const py::array_t& point_ids, - const py::dict& centers) { + const py::dict& frame_centers) { MatrixX3d camera_values(image_ids.size(), 3), point_values(point_ids.size(), 3); for (size_t i = 0; i < image_ids.size(); ++i) { - const Eigen::Vector3d center = - centers.empty() - ? reconstruction.Image(image_ids[i]).ProjectionCenter() - : centers[py::int_(image_ids[i])].cast(); + const auto& image = reconstruction.Image(image_ids[i]); + Eigen::Vector3d center = + frame_centers.empty() ? image.ProjectionCenter() + : frame_centers[py::int_(image.FrameId())] + .cast(); + if (!frame_centers.empty() && !image.IsRefInFrame()) + center -= RigTranslationInWorld(image); camera_values.row(i) = center.transpose(); } for (py::ssize_t i = 0; i < point_ids.size(); ++i) @@ -102,7 +106,7 @@ void BindMappingSidecars(py::module_& m) { py::arg("reconstruction"), py::arg("image_ids"), py::arg("point_ids"), - py::arg("centers") = py::dict()); + py::arg("frame_centers") = py::dict()); m.def( "publish_geometry", [](const colmap::Reconstruction& source, diff --git a/extensions/colmap/src/bindings/tracks.cc b/extensions/colmap/src/bindings/tracks.cc index 04a859d..874d698 100644 --- a/extensions/colmap/src/bindings/tracks.cc +++ b/extensions/colmap/src/bindings/tracks.cc @@ -1,13 +1,79 @@ #include "vidmap_native/tracks.h" +#include "colmap/scene/correspondence_graph.h" + +#include +#include + #include "bindings.h" +#include #include namespace py = pybind11; namespace vidmap { +// Keep COLMAP traversal in native code and materialize each pair once. +py::dict BuildTransitiveCorrespondences( + const colmap::CorrespondenceGraph& graph, + const std::vector& images, + const std::vector& counts, + size_t depth) { + if (images.size() != counts.size()) + throw std::invalid_argument("image and observation inventories differ"); + struct Entry { + colmap::image_t first, second; + std::vector> rows; + std::unordered_set unique; + }; + std::vector entries; + { + py::gil_scoped_release release; + std::unordered_map indices; + std::vector corrs; + for (size_t i = 0; i < images.size(); ++i) { + const auto image = images[i]; + if (!graph.ExistsImage(image)) continue; + for (colmap::point2D_t point = 0; point < counts[i]; ++point) { + graph.ExtractTransitiveCorrespondences(image, point, depth, &corrs); + for (const auto& corr : corrs) { + const uint64_t forward = (uint64_t(image) << 32) | corr.image_id; + const uint64_t reverse = (uint64_t(corr.image_id) << 32) | image; + if (indices.count(reverse)) continue; + const auto [it, inserted] = indices.emplace(forward, entries.size()); + if (inserted) entries.push_back({image, corr.image_id, {}, {}}); + auto& entry = entries[it->second]; + const uint64_t row = (uint64_t(point) << 32) | corr.point2D_idx; + if (entry.unique.insert(row).second) + entry.rows.emplace_back(point, corr.point2D_idx); + } + } + } + for (auto& entry : entries) + std::unordered_set().swap(entry.unique); + } + py::dict result; + for (const auto& entry : entries) { + // Preserve the established Python tuple-set row ordering exactly. + py::set unique; + for (const auto& row : entry.rows) + unique.add(py::make_tuple(row.first, row.second)); + py::array_t array( + {static_cast(entry.rows.size()), py::ssize_t(2)}); + auto rows = array.mutable_unchecked<2>(); + py::ssize_t i = 0; + for (const auto& row : unique) { + const auto pair = py::cast>(row); + rows(i, 0) = pair.first; + rows(i++, 1) = pair.second; + } + result[py::make_tuple(entry.first, entry.second)] = std::move(array); + } + return result; +} + void BindTracks(py::module_& m) { + m.def("build_transitive_correspondences", &BuildTransitiveCorrespondences); py::class_(m, "TrackEstablishmentOptions") .def(py::init<>()) .def_readwrite( diff --git a/extensions/colmap/src/stages/depth_prior.h b/extensions/colmap/src/stages/depth_prior.h index 59ac0f5..71e0676 100644 --- a/extensions/colmap/src/stages/depth_prior.h +++ b/extensions/colmap/src/stages/depth_prior.h @@ -2,6 +2,7 @@ #include "colmap/estimators/cost_functions/quaternion_utils.h" #include "colmap/estimators/cost_functions/utils.h" +#include "colmap/geometry/rigid3.h" #include #include @@ -11,16 +12,28 @@ namespace vidmap { struct LogScaledDepthErrorCostFunctor : public colmap:: AutoDiffCostFunctor { - explicit LogScaledDepthErrorCostFunctor(double depth) : depth_(depth) {} + explicit LogScaledDepthErrorCostFunctor( + double depth, const colmap::Rigid3d& sensor_from_rig = colmap::Rigid3d()) + : depth_(depth), + sensor_z_axis_(sensor_from_rig.rotation().toRotationMatrix().row(2)), + sensor_z_offset_(sensor_from_rig.translation().z()), + direct_depth_(sensor_z_axis_ == Eigen::Vector3d::UnitZ() && + sensor_z_offset_ == 0.0) {} template bool operator()(const T* pose, const T* point3D, const T* log_scale, T* residuals) const { - const T predicted_depth = (colmap::EigenQuaternionMap(pose) * - colmap::EigenVector3Map(point3D))[2] + - pose[6]; + const Eigen::Matrix rotated_point = + colmap::EigenQuaternionMap(pose) * + colmap::EigenVector3Map(point3D); + const T predicted_depth = + direct_depth_ + ? rotated_point[2] + pose[6] + : sensor_z_axis_.cast().dot( + rotated_point + colmap::EigenVector3Map(pose + 4)) + + T(sensor_z_offset_); if (predicted_depth <= T(0.0)) { residuals[0] = T(0.0); return true; @@ -32,6 +45,9 @@ struct LogScaledDepthErrorCostFunctor private: const double depth_; + const Eigen::Vector3d sensor_z_axis_; + const double sensor_z_offset_; + const bool direct_depth_; }; } // namespace vidmap diff --git a/extensions/colmap/src/stages/image_pair_inliers.cc b/extensions/colmap/src/stages/image_pair_inliers.cc index cdaad4d..fc28260 100644 --- a/extensions/colmap/src/stages/image_pair_inliers.cc +++ b/extensions/colmap/src/stages/image_pair_inliers.cc @@ -215,6 +215,22 @@ void ImagePairsInlierCount(const InlierThresholdOptions& options, if (!graph.IsValid(pair_id)) { continue; } + const auto [first, second] = colmap::PairIdToImagePair(pair_id); + if (reconstruction.Image(first).FrameId() == + reconstruction.Image(second).FrameId()) { + // Fixed stereo geometry takes precedence over a fitted F/H + // classification. Retain that classification and the original row/LC + // metadata unchanged. + const auto& pose = graph.Edges().at(pair_id).cam2_from_cam1; + if (!pose.translation().allFinite() || + pose.translation().norm() <= kEpsilon) { + throw std::invalid_argument( + "same-frame scoring requires finite nonzero calibrated translation"); + } + ScoreEssentialPair( + options, reconstruction, graph, sidecars, pair_id, &pair); + continue; + } switch (pair.geometry.config) { case colmap::TwoViewGeometry::CALIBRATED: ScoreEssentialPair( diff --git a/extensions/colmap/src/stages/metric_depth.h b/extensions/colmap/src/stages/metric_depth.h index 8328e1a..c878bd3 100644 --- a/extensions/colmap/src/stages/metric_depth.h +++ b/extensions/colmap/src/stages/metric_depth.h @@ -14,14 +14,17 @@ namespace vidmap { struct MetricDepthError : public colmap::AutoDiffCostFunctor { - MetricDepthError(const Eigen::Quaterniond& rotation, - double depth_prior, - double sigma_depth, - bool use_log_scale, - MetricDepthResidualType residual_type, - bool zero_residual_behind, - double log_linear_threshold) + MetricDepthError( + const Eigen::Quaterniond& rotation, + double depth_prior, + double sigma_depth, + bool use_log_scale, + MetricDepthResidualType residual_type, + bool zero_residual_behind, + double log_linear_threshold, + const Eigen::Vector3d& rig_translation = Eigen::Vector3d::Zero()) : rotation_(rotation), + rig_translation_(rig_translation), depth_prior_(depth_prior), sigma_depth_(sigma_depth), use_log_scale_(use_log_scale), @@ -45,8 +48,10 @@ struct MetricDepthError using Vector3 = Eigen::Matrix; const Vector3 point_world = Eigen::Map(point3D); const Vector3 center_world = Eigen::Map(camera_center); - const T estimated_depth = - (rotation_.cast() * (point_world - center_world))[2]; + Vector3 displacement = point_world - center_world; + if (!rig_translation_.isZero(0.0)) + displacement += rig_translation_.cast(); + const T estimated_depth = (rotation_.cast() * displacement)[2]; const T scale = use_log_scale_ ? ceres::exp(depth_map_scale[0]) : depth_map_scale[0]; const T scaled_prior = scale * T(depth_prior_); @@ -91,6 +96,7 @@ struct MetricDepthError } const Eigen::Quaterniond rotation_; + const Eigen::Vector3d rig_translation_; const double depth_prior_; const double sigma_depth_; const bool use_log_scale_; diff --git a/extensions/colmap/src/stages/rig_geometry.h b/extensions/colmap/src/stages/rig_geometry.h new file mode 100644 index 0000000..c91761a --- /dev/null +++ b/extensions/colmap/src/stages/rig_geometry.h @@ -0,0 +1,42 @@ +#pragma once + +#include "colmap/scene/reconstruction.h" + +#include +#include + +namespace vidmap { + +inline colmap::Rigid3d FixedSensorFromRig(const colmap::Image& image) { + if (image.IsRefInFrame()) return colmap::Rigid3d(); + const auto& rig = *image.FramePtr()->RigPtr(); + const auto sensor_id = image.CameraPtr()->SensorId(); + if (!rig.HasSensorFromRig(sensor_id)) + throw std::invalid_argument("solver requires known fixed rig extrinsics"); + const auto& pose = rig.SensorFromRig(sensor_id); + if (!pose.rotation().coeffs().allFinite() || + std::abs(pose.rotation().norm() - 1.0) > 1e-8 || + !pose.translation().allFinite()) + throw std::invalid_argument( + "solver requires finite rigid sensor transforms"); + return pose; +} + +// Positive offset added to X - C_rig in the fixed-rig BATA convention. +inline Eigen::Vector3d RigTranslationInWorld(const colmap::Image& image) { + if (image.IsRefInFrame()) return Eigen::Vector3d::Zero(); + return image.CamFromWorld().rotation().inverse() * + FixedSensorFromRig(image).translation(); +} + +inline bool ValidateFixedRigScene( + const colmap::Reconstruction& reconstruction) { + bool is_rig = false; + for (const auto& [id, image] : reconstruction.Images()) { + FixedSensorFromRig(image); + is_rig |= !image.IsRefInFrame(); + } + return is_rig; +} + +} // namespace vidmap diff --git a/extensions/colmap/src/stages/track_establishment.cc b/extensions/colmap/src/stages/track_establishment.cc index 5776f57..030511b 100644 --- a/extensions/colmap/src/stages/track_establishment.cc +++ b/extensions/colmap/src/stages/track_establishment.cc @@ -1,16 +1,19 @@ // Track establishment over explicit regular and loop-closure observations. +#include "colmap/geometry/triangulation.h" #include "colmap/math/union_find.h" #include "colmap/scene/two_view_geometry.h" #include #include #include +#include #include #include #include #include #include +#include "stages/rig_geometry.h" #include "vidmap_native/tracks.h" namespace vidmap { @@ -302,7 +305,51 @@ TrackMap EstablishTracksFromCorrGraph( return tracks; } +// Two calibrated sensors at one exposure provide metric stereo support even +// when neither observation has a monocular depth prior. Temporal two-view +// tracks still use the original depth gate. +bool HasTriangulatableStereo(const colmap::Reconstruction& reconstruction, + const NativeTrack& track) { + std::map features; + const auto insert = [&](const TrackElement& observation) { + const auto [it, added] = + features.emplace(observation.image_id, observation.point2D_idx); + return added || it->second == observation.point2D_idx; + }; + for (const auto& observation : track.observations) + if (!insert(observation)) return false; + for (const auto& observation : track.loop_closure_observations) + if (!insert(observation.observation)) return false; + if (features.size() != 2) return false; + const auto first = features.begin(); + const auto second = std::next(first); + const auto& image1 = reconstruction.Image(first->first); + const auto& image2 = reconstruction.Image(second->first); + if (image1.FrameId() != image2.FrameId() || + image1.CameraId() == image2.CameraId() || + !image1.CameraPtr()->HasPriorFocalLength() || + !image2.CameraPtr()->HasPriorFocalLength()) + return false; + const auto relative = + FixedSensorFromRig(image2) * colmap::Inverse(FixedSensorFromRig(image1)); + if (relative.translation().norm() <= 1e-6) return false; + const auto ray1 = + image1.CameraPtr()->CamRayFromImg(image1.Point2D(first->second).xy); + const auto ray2 = + image2.CameraPtr()->CamRayFromImg(image2.Point2D(second->second).xy); + if (!ray1 || !ray2 || !ray1->allFinite() || !ray2->allFinite()) return false; + if ((relative.rotation() * *ray1).cross(*ray2).norm() <= 1e-6) return false; + Eigen::Vector3d point; + if (!colmap::TriangulateMidPoint(relative, *ray1, *ray2, &point) || + !point.allFinite()) + return false; + // Signed distance along the actual rays also works for non-perspective + // cameras; a positive camera-z test would incorrectly reject rear rays. + return point.dot(*ray1) > 1e-6 && (relative * point).dot(*ray2) > 1e-6; +} + TrackMap FilterTracksForProblem( + const colmap::Reconstruction& reconstruction, const MappingSidecars& sidecars, const std::vector& registered_image_ids, const TrackMap& tracks_full, @@ -350,7 +397,9 @@ TrackMap FilterTracksForProblem( [&](const LoopClosureObservation& observation) { return HasValidDepthPrior(sidecars, observation.observation); }); - if (!regular_depths_valid || !loop_closure_depths_valid) continue; + if ((!regular_depths_valid || !loop_closure_depths_valid) && + !HasTriangulatableStereo(reconstruction, candidate)) + continue; } selected.emplace(track_id, std::move(candidate)); } @@ -376,7 +425,8 @@ TrackEstablishmentResult EstablishAndCommitTracks( pair_order, options, loop_closure_second_pass); - auto selected = FilterTracksForProblem(sidecars, image_order, full, options); + auto selected = FilterTracksForProblem( + reconstruction, sidecars, image_order, full, options); TrackEstablishmentResult result; result.num_full_tracks = full.size(); result.num_tracks = selected.size(); diff --git a/extensions/colmap/tests/python/test_global_positioning.py b/extensions/colmap/tests/python/test_global_positioning.py index 635ab40..dc9bfb2 100644 --- a/extensions/colmap/tests/python/test_global_positioning.py +++ b/extensions/colmap/tests/python/test_global_positioning.py @@ -6,7 +6,6 @@ import pytest import vidmap_native._core as native -from tests.mapper.test_bundle_adjustment_problem import _options as _ba_options from vidmap.mapper.native.state import SolveState from vidmap.mapper.options.positioning import GPOptions from vidmap.mapper.playback_trace import SolverPlaybackOptions @@ -15,6 +14,26 @@ from vidmap.mapper.stages.global_positioning.problem import TemporalAccelerationPrior +def _ba_options(state, *, num_threads=1): + from vidmap.mapper.options.solver import SolverBackendOptions + from vidmap.mapper.stages.bundle_adjustment.options import build_bundle_adjustment_options + + return build_bundle_adjustment_options( + reconstruction=state.reconstruction, + image_order=state.image_order, + camera_ids=[], + variable_point3D_ids=[], + optimize_intrinsics=True, + refine_principal_point=False, + fix_rotations=False, + reprojection_loss="trivial", + reprojection_scale=1.0, + reprojection_weight=1.0, + num_threads=num_threads, + solver_backend=SolverBackendOptions(), + ) + + def _gp_options(rounds=0): return GPOptions( common=dict( @@ -32,7 +51,10 @@ def _gp_options(rounds=0): def _run_gp(options, state, callback=None, *, timeline=None, **kwargs): prepared = gp.prepare_observations( - state.reconstruction, state.sidecars, options, state.image_order if timeline is None else timeline + state.reconstruction, + state.sidecars, + options, + tuple(dict.fromkeys(state.image(i).frame_id for i in state.image_order)) if timeline is None else timeline, ) create = pycolmap.create_default_global_positioner @@ -110,7 +132,7 @@ def test_continuous_support_reduces_cost(): def test_sequential_support_requires_complete_unique_timeline(): - with pytest.raises(ValueError, match="unique image timeline"): + with pytest.raises(ValueError, match="unique frame timeline"): _run_gp(_gp_options(2), _playback_problem([1, 2, 3]), timeline=[1, 2]) diff --git a/extensions/colmap/tests/python/test_rotation_averaging.py b/extensions/colmap/tests/python/test_rotation_averaging.py index d7641ba..4fc71a4 100644 --- a/extensions/colmap/tests/python/test_rotation_averaging.py +++ b/extensions/colmap/tests/python/test_rotation_averaging.py @@ -42,7 +42,8 @@ def solve(problem, sequence_ids=None, **option_values): return set(problem[0].reg_image_ids()) -def test_disconnected_graph_solves_only_the_largest_component(): +@pytest.mark.parametrize("threshold", [0.0, 2.0]) +def test_disconnected_graph_preserves_dev_output_policy(threshold): image_ids = [1, 2, 3, 4, 5] edges = [ (1, 2, [0.0, 0.04, 0.0]), @@ -51,10 +52,13 @@ def test_disconnected_graph_solves_only_the_largest_component(): ] problem = rotation_problem(image_ids, set(image_ids), edges) - before = {i: problem[0].image(i).cam_from_world().matrix().copy() for i in (4, 5)} - solve(problem) - for i, pose in before.items(): - np.testing.assert_array_equal(problem[0].image(i).cam_from_world().matrix(), pose) + before = {i: problem[0].image(i).cam_from_world().params.copy() for i in (4, 5)} + assert solve(problem, max_rotation_error_deg=threshold) == ({1, 2, 3} if threshold else set(image_ids)) + assert all(edge.valid for edge in problem[1].edges.values()) + for image_id, pose in before.items(): + assert problem[0].image(image_id).has_pose == (threshold == 0) + if threshold == 0: + np.testing.assert_array_equal(problem[0].image(image_id).cam_from_world().params, pose) for a, b, angle_axis in edges[:2]: relative = ( problem[0].image(b).cam_from_world().rotation * problem[0].image(a).cam_from_world().rotation.inverse() @@ -129,12 +133,18 @@ def test_vidmap_losses_and_tracking_initialization(monkeypatch, skip_loops): def capture(options, local_graph, reconstruction): losses.append(options.ceres.loss_function_type) assert set(local_graph.edges) == set(graph.edges) - {loop_id} + assert options.skip_initialization == (len(losses) > 1) + assert not options.refine_sensor_from_rig rotations = {i: image.cam_from_world().rotation.matrix().copy() for i, image in reconstruction.images.items()} owner = create(options, local_graph, reconstruction) assert owner.problem.num_residual_blocks() == 2 - for image_id, image in reconstruction.images.items(): - expected = rotations[image_id] if options.skip_initialization else np.eye(3) - np.testing.assert_allclose(image.cam_from_world().rotation.matrix(), expected, atol=1e-12) + if options.skip_initialization: + for image_id, image in reconstruction.images.items(): + np.testing.assert_allclose(image.cam_from_world().rotation.matrix(), rotations[image_id], atol=1e-12) + # Even though the loop is inconsistent, tracking edges initialize all rotations to identity. + if len(losses) == 1: + for image in reconstruction.images.values(): + np.testing.assert_allclose(image.cam_from_world().rotation.matrix(), np.eye(3), atol=1e-12) class Solve: problem = owner.problem @@ -178,8 +188,13 @@ def test_loop_closures_bridge_nearest_temporal_gaps(monkeypatch, skip_loops, all if all_loops or (first, second) != (1, 2): pair.are_loop_closure = np.ones(10, dtype=np.uint8) create = pycolmap.create_default_ceres_rotation_averager + graphs = [] def capture(options, local_graph, reconstruction): + if graphs: + assert local_graph is graphs[0] + graphs.append(local_graph) + assert options.skip_initialization == (len(graphs) > 1) assert {pid for pid, edge in local_graph.edges.items() if edge.valid} == { pycolmap.image_pair_to_pair_id(a, b) for a, b in [(1, 2), (2, 3), (1, 4)] } @@ -190,6 +205,7 @@ def capture(options, local_graph, reconstruction): np.testing.assert_allclose(reconstruction.image(4).cam_from_world().rotation.matrix(), expected, atol=1e-12) class Solve: + problem = owner.problem add_relative_rotation_residual = owner.add_relative_rotation_residual def solve(self): @@ -215,7 +231,9 @@ def solve(self): monkeypatch.setattr(pycolmap, "create_default_ceres_rotation_averager", capture) # Pair (1, 4) is closer in time than (3, 4), despite fewer inliers and a larger ID gap. solve(problem, sequence_ids=[3, 2, 1, 4, 5, 6], filter_risky_loop_closure_pairs=skip_loops) + assert len(graphs) == 2 assert all(edge.num_matches == 0 for edge in graph.edges.values()) + assert all(edge.valid for edge in graph.edges.values()) def test_filtering_every_edge_fails(): diff --git a/vidmap/calibration.py b/vidmap/calibration.py new file mode 100644 index 0000000..5a00d8e --- /dev/null +++ b/vidmap/calibration.py @@ -0,0 +1,57 @@ +"""Original prediction associations shared by extraction and mapping.""" + +from dataclasses import asdict, dataclass + + +@dataclass(frozen=True) +class CalibrationObservation: + camera_id: int + key: str + image_name: str | None + + +@dataclass(frozen=True) +class CalibrationEvidence: + estimator: str + inference: str + model: str + pixel_domain: str + observations: tuple[CalibrationObservation, ...] + + def __post_init__(self): + if self.estimator not in {"geocalib", "da3"}: + raise ValueError("Calibration evidence requires an enabled predictor") + if self.inference not in {"selected_batch", "per_view"}: + raise ValueError("Unknown calibration inference") + if self.model != "PINHOLE" or self.pixel_domain != "working_images": + raise ValueError("Unsupported calibration prediction domain") + if self.estimator == "da3" and (self.inference != "per_view" or self.model != "PINHOLE"): + raise ValueError("DA3 calibration requires per-view pinhole predictions") + if not self.observations: + raise ValueError("Calibration results require camera associations") + if len(set(self.observations)) != len(self.observations): + raise ValueError("Repeated calibration observation") + shared = self.inference == "selected_batch" + if any((observation.image_name is None) != shared for observation in self.observations): + raise ValueError("Calibration observations disagree with inference mode") + if not shared and len({observation.image_name for observation in self.observations}) != len(self.observations): + raise ValueError("Per-view calibration requires unique image associations") + + def as_dict(self) -> dict: + return { + "estimator": self.estimator, + "inference": self.inference, + "model": self.model, + "pixel_domain": self.pixel_domain, + "observations": [asdict(observation) for observation in self.observations], + } + + @classmethod + def from_dict(cls, value: dict) -> "CalibrationEvidence": + return cls( + estimator=value["estimator"], + inference=value["inference"], + model=value["model"], + pixel_domain=value["pixel_domain"], + observations=tuple(CalibrationObservation(**item) for item in value["observations"]), + ) diff --git a/vidmap/configs/frontend/calib/rig.yaml b/vidmap/configs/frontend/calib/rig.yaml new file mode 100644 index 0000000..6901e69 --- /dev/null +++ b/vidmap/configs/frontend/calib/rig.yaml @@ -0,0 +1,10 @@ +defaults: + - base + +keyframes: + matching: + coarse_only: true +tracks: + propagation: + max_kps: 1000 +rig: {} diff --git a/vidmap/configuration/config.py b/vidmap/configuration/config.py index b797b65..adbabee 100644 --- a/vidmap/configuration/config.py +++ b/vidmap/configuration/config.py @@ -20,6 +20,8 @@ PreparationOptions, RoMaV2Options, ) +from vidmap.frontend.options.preparation import CameraProcessingOptions +from vidmap.frontend.options.rig import CameraId, RigOptions from vidmap.mapper.options import MapperOptions @@ -40,6 +42,10 @@ class FrontendConfig: depth: DepthEstimationOptions = dc_field(default_factory=DepthEstimationOptions) camera_priors: CameraPriorEstimationOptions = dc_field(default_factory=CameraPriorEstimationOptions) preparation: PreparationOptions = dc_field(default_factory=PreparationOptions) + rig: RigOptions | None = dc_field(default=None, metadata={"omit_default": True}) + camera_processing: dict[CameraId, CameraProcessingOptions] = dc_field( + default_factory=dict, metadata={"omit_default": True} + ) @model_validator(mode="before") @classmethod @@ -48,6 +54,8 @@ def _normalize_input(cls, values): if not isinstance(values, dict): return values values = dict(values) + if isinstance(values.get("rig"), dict): + values["rig"] = RigOptions(**values["rig"]) for name, option_type in { "roma": RoMaV2Options, "keyframes": KeyframeOptions, diff --git a/vidmap/configuration/dump.py b/vidmap/configuration/dump.py index eea7833..f89acba 100644 --- a/vidmap/configuration/dump.py +++ b/vidmap/configuration/dump.py @@ -34,6 +34,12 @@ def config_to_dict(obj): values = {} for field in dataclasses.fields(obj): value = getattr(obj, field.name) + if field.metadata.get("omit_default", False): + default = field.default + if default is dataclasses.MISSING and field.default_factory is not dataclasses.MISSING: + default = field.default_factory() + if value == default: + continue values[field.name] = config_to_dict(value) return values if isinstance(obj, dict): diff --git a/vidmap/datasets/capture.py b/vidmap/datasets/capture.py new file mode 100644 index 0000000..876a819 --- /dev/null +++ b/vidmap/datasets/capture.py @@ -0,0 +1,363 @@ +# Strict integer checks intentionally reject bool. +# ruff: noqa: E721 +"""Frontend captures with native COLMAP geometry and optional frame times.""" + +from __future__ import annotations + +import csv +import hashlib +import json +import math +from dataclasses import dataclass, field +from functools import cached_property +from pathlib import Path +from typing import TYPE_CHECKING, Literal, Mapping + +from PIL import Image + +from vidmap.datasets.frame_names import timestamp_from_image_name, timestamp_ns_from_image_name +from vidmap.repro.hashing import sha256_file + +if TYPE_CHECKING: + import pycolmap + + +def _id(value: object, label: str) -> int: + if type(value) is not int or not 0 <= value < 2**32 - 1: + raise ValueError(f"{label} must be a nonnegative COLMAP integer ID") + return value + + +def _name(value: object) -> str: + if not isinstance(value, str) or not value or "\\" in value or "\x00" in value: + raise ValueError("Image names must be nonempty relative POSIX paths") + if Path(value).is_absolute() or any(part in ("", ".", "..") for part in value.split("/")): + raise ValueError(f"Unsafe image name: {value!r}") + return value + + +def _fingerprint(value) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + ).hexdigest() + + +def calibration_fingerprint(rec) -> str: + """Identify native camera calibration without duplicating its persisted graph.""" + return _fingerprint(_calibration(rec)) + + +def _validate_camera_frame(rec, frame): + import pycolmap + + sensors = set(rec.rig(frame.rig_id).sensor_ids()) + if any(data.sensor_id.type != pycolmap.SensorType.CAMERA for data in frame.data_ids): + raise ValueError("VidMap capture frames currently support camera sensors only") + if len(frame.image_ids) != len(sensors) or {data.sensor_id for data in frame.image_ids} != sensors: + raise ValueError("Incomplete capture frame; supply every camera exactly once") + if any(rec.image(data.id).frame_id != frame.frame_id for data in frame.image_ids): + raise ValueError("COLMAP frame and image inventories disagree") + + +@dataclass +class CaptureSnapshot: + """Validated source bytes for one frontend execution; geometry remains native.""" + + capture: PreparedCapture + image_sha256: Mapping[str, str] + + def __post_init__(self): + self.image_sha256 = dict(self.image_sha256) + + def image_content_fingerprint(self, names) -> str: + # Same canonical JSON as the frontend cache contract, without rereading media. + return _fingerprint([[name, self.image_sha256[name]] for name in names]) + + def fingerprint(self) -> str: + capture = self.capture + identity = _fingerprint( + { + "images": dict(self.image_sha256), + "calibration": calibration_fingerprint(capture.rec), + "membership": [ + [ + image.image_id, + image.name, + image.camera_id, + image.frame_id, + capture.rec.frame(image.frame_id).rig_id, + ] + for _, image in sorted(capture.rec.images.items()) + ], + "frame_order": capture.frame_ids, + "frame_timestamps_ns": ( + None if capture.frame_timestamps_ns is None else dict(capture.frame_timestamps_ns) + ), + "preserve_calibration": capture.preserve_calibration, + "forced_keyframes": sorted(capture.forced_keyframe_names), + } + ) + if "source_fingerprint" in capture.provenance: + identity = _fingerprint([identity, capture.provenance["source_fingerprint"]]) + return identity + + +@dataclass +class PreparedCapture: + """Native capture geometry, borrowed read-only throughout a frontend execution.""" + + rec: pycolmap.Reconstruction + rgb_dir: Path + frame_timestamps_ns: Mapping[int, int] | None + kind: Literal["rig"] | None + provenance: Mapping + preserve_calibration: bool = True + scene: str = "capture" + forced_keyframe_names: frozenset[str] = frozenset() + _image_ids_by_name: Mapping[str, int] = field(init=False, repr=False, compare=False) + + def __post_init__(self): + if not self.rec.is_valid(): + raise ValueError("Invalid native capture reconstruction") + if not self.rec.cameras or not self.rec.images: + raise ValueError("A capture requires cameras and images") + if len(self.rec.rigs) > 1 and any(rig.num_sensors() != 1 for rig in self.rec.rigs.values()): + raise ValueError("A capture must have one fixed rig or a sequence of singleton frames") + sensors = {sensor for rig in self.rec.rigs.values() for sensor in rig.sensor_ids()} + if sensors != {camera.sensor_id for camera in self.rec.cameras.values()}: + raise ValueError("COLMAP rig sensors contradict capture cameras") + for rig in self.rec.rigs.values(): + for sensor in rig.non_ref_sensors: + transform = rig.sensor_from_rig(sensor) + if transform is None or not all(math.isfinite(value) for row in transform.matrix() for value in row): + raise ValueError("Every sensor requires a finite fixed transform") + if self.frame_timestamps_ns is not None: + times = self.frame_timestamps_ns + if not isinstance(times, Mapping) or set(times) != set(self.rec.frames): + raise ValueError("Capture timestamps must cover exactly the native frame IDs") + if any(type(key) is not int or type(value) is not int for key, value in times.items()): + raise ValueError("Capture timestamps and IDs must be integers") + if len(set(self.frame_timestamps_ns.values())) != len(self.rec.frames): + raise ValueError("Capture frame timestamps must increase") + else: + if any(rig.num_sensors() != 1 for rig in self.rec.rigs.values()): + raise ValueError("Multi-camera captures require explicit timestamps") + for frame in self.rec.frames.values(): + _validate_camera_frame(self.rec, frame) + if any(frame.has_pose() for frame in self.rec.frames.values()): + raise ValueError("Capture inputs must be unposed; GT is evaluation-only") + image_ids = {_name(image.name): image.image_id for image in self.rec.images.values()} + if len(image_ids) != len(self.rec.images): + raise ValueError("Capture image names must be unique") + self._image_ids_by_name = image_ids + self.rgb_dir = Path(self.rgb_dir) + if self.frame_timestamps_ns is not None: + self.frame_timestamps_ns = dict(self.frame_timestamps_ns) + self.provenance = dict(self.provenance) + + def image(self, name: str) -> pycolmap.Image: + """Resolve names in constant time; membership edits require a new capture.""" + image = self.rec.image(self._image_ids_by_name[name]) + if image.name != name: + raise ValueError("Capture image names changed after preparation") + return image + + @cached_property + def frame_ids(self) -> tuple[int, ...]: + if self.frame_timestamps_ns is not None: + return tuple(sorted(self.rec.frames, key=self.frame_timestamps_ns.__getitem__)) + images = sorted(self.rec.images.values(), key=lambda image: image.image_id) + return tuple( + image.frame_id for image in sorted(images, key=lambda image: timestamp_from_image_name(image.name)) + ) + + @property + def reference_camera_id(self) -> int: + return next(iter(self.rec.rigs.values())).ref_sensor_id.id + + @property + def imnames(self) -> tuple[str, ...]: + return tuple(self.rec.image(data.id).name for fid in self.frame_ids for data in self.rec.frame(fid).image_ids) + + def snapshot(self) -> CaptureSnapshot: + """Hash source bytes and validate prepared images before cache reuse.""" + images = {} + root = self.rgb_dir.resolve(strict=True) + for name in self.imnames: + source = root / name + if self.kind == "rig": + source = source.resolve(strict=True) + if not source.is_relative_to(root): + raise ValueError("Capture image escapes the image directory") + camera = self.rec.camera(self.image(name).camera_id) + with Image.open(source) as pixels: + if pixels.size != (camera.width, camera.height): + raise ValueError(f"Capture image dimensions changed: {name}") + images[name] = sha256_file(source) + expected_images = self.provenance["image_sha256"] if "image_sha256" in self.provenance else {} + for name, expected in expected_images.items(): + if name not in images or images[name] != expected: + raise ValueError(f"Prepared capture image changed: {name}") + return CaptureSnapshot(self, images) + + +def read_frame_table(path: Path) -> tuple[dict, ...]: + """Read explicit CSV associations without inventing another geometry model.""" + columns = {"frame_id", "timestamp_ns", "camera_id", "image_name"} + rows, frame_times, names, members = [], {}, set(), set() + with path.open(newline="", encoding="utf-8") as stream: + reader = csv.DictReader(stream) + if reader.fieldnames is None or len(reader.fieldnames) != len(columns) or set(reader.fieldnames) != columns: + raise ValueError("Frame CSV must contain exactly the required columns") + for row in reader: + item = {key: _name(value) if key == "image_name" else int(value) for key, value in row.items()} + fid, cid = _id(item["frame_id"], "Frame ID"), _id(item["camera_id"], "Camera ID") + timestamp = item["timestamp_ns"] + if fid in frame_times and frame_times[fid] != timestamp: + raise ValueError(f"Inconsistent timestamps for frame {fid}") + if fid not in frame_times: + if frame_times and timestamp <= next(reversed(frame_times.values())): + raise ValueError("Capture frame timestamps must increase") + frame_times[fid] = timestamp + if item["image_name"] in names or (fid, cid) in members: + raise ValueError("Capture images and frame/camera memberships must be unique") + names.add(item["image_name"]) + members.add((fid, cid)) + rows.append(item) + if not rows: + raise ValueError("Frame CSV must not be empty") + return tuple(rows) + + +def resolve_capture_frames(root: Path, directories: tuple[str, ...], frames: str | None) -> tuple[dict, ...]: + """Read explicit associations, or synchronize streams by filename timestamps.""" + from vidmap.datasets.local import select_images + + if frames is not None: + return read_frame_table(root / frames) + streams = [] + for directory in directories: + stream = {} + for name in select_images(root / directory, None): + image_name = (Path(directory) / name).as_posix() + timestamp = timestamp_ns_from_image_name(image_name) + if timestamp in stream: + raise ValueError(f"Duplicate timestamp in {directory!r}; set rig.frames to a frames.csv") + stream[timestamp] = image_name + streams.append(stream) + timestamps = sorted(streams[0]) + if any(stream.keys() != streams[0].keys() for stream in streams[1:]): + raise ValueError("Camera timestamps do not match exactly; set rig.frames to a frames.csv") + return tuple( + dict(frame_id=fid, timestamp_ns=timestamp, camera_id=cid, image_name=stream[timestamp]) + for fid, timestamp in enumerate(timestamps, start=1) + for cid, stream in enumerate(streams) + ) + + +def build_capture_reconstruction(cameras, rig, rows): + """Associate explicit rows with native frames, keeping timing in sidecar maps.""" + import pycolmap + + rec = pycolmap.Reconstruction() + for camera in cameras: + rec.add_camera(camera) + rec.add_rig(rig) + grouped = {} + for row in rows: + grouped.setdefault(row["frame_id"], []).append(row) + frame_times = {} + image_id = 1 + for fid, entries in grouped.items(): + frame = pycolmap.Frame(frame_id=fid, rig_id=rig.rig_id) + images = [] + for entry in sorted(entries, key=lambda row: row["camera_id"]): + frame_times[fid] = entry["timestamp_ns"] + image = pycolmap.Image( + image_id=image_id, name=entry["image_name"], camera_id=entry["camera_id"], frame_id=fid + ) + frame.add_data_id(image.data_id) + images.append(image) + image_id += 1 + rec.add_frame(frame) + for image in images: + rec.add_image(image) + return rec, frame_times + + +def _calibration(rec: pycolmap.Reconstruction) -> dict: + return { + "cameras": { + str(camera_id): { + "model": camera.model_name, + "width": camera.width, + "height": camera.height, + "params": camera.params.tolist(), + "has_prior_focal_length": camera.has_prior_focal_length, + } + for camera_id, camera in sorted(rec.cameras.items()) + }, + "rigs": { + str(rig_id): { + "reference": rig.ref_sensor_id.id, + "sensors": { + str(sensor.id): ( + None if rig.is_ref_sensor(sensor) else rig.sensor_from_rig(sensor).matrix().tolist() + ) + for sensor in sorted(rig.sensor_ids(), key=lambda sensor: sensor.id) + }, + } + for rig_id, rig in sorted(rec.rigs.items()) + }, + } + + +def prepare_scene_capture(scene_parser) -> PreparedCapture: + """Adapt a dataset's calibration and image inventory to the frontend capture.""" + if isinstance(scene_parser, PreparedCapture): + return scene_parser + + return PreparedCapture( + rec=copy_unposed_reconstruction(scene_parser.rec), + rgb_dir=scene_parser.rgb_dir, + frame_timestamps_ns=None, + kind=None, + provenance={}, + preserve_calibration=False, + forced_keyframe_names=frozenset(image.name for image in scene_parser.rec.images.values() if image.has_pose), + ) + + +def copy_unposed_reconstruction(source, reference_image_names=None) -> pycolmap.Reconstruction: + """Copy image calibration and complete rig frames, excluding all input poses.""" + import pycolmap + + names = None if reference_image_names is None else set(reference_image_names) + images = [image for image in source.images.values() if names is None or image.name in names] + if names is not None and {image.name for image in images} != names: + raise ValueError("Initial reconstruction selects unknown image names") + selected_ids = {image.image_id for image in images} + frame_ids = tuple(dict.fromkeys(image.frame_id for image in images)) + for frame_id in frame_ids: + frame = source.frame(frame_id) + if {data.id for data in frame.image_ids} - selected_ids: + raise ValueError("Rig image selection must retain complete frames") + _validate_camera_frame(source, frame) + rec = pycolmap.Reconstruction() + for camera_id in dict.fromkeys(image.camera_id for image in images): + rec.add_camera(source.camera(camera_id)) + for rig_id in dict.fromkeys(source.frame(frame_id).rig_id for frame_id in frame_ids): + rec.add_rig(source.rig(rig_id)) + for frame_id in frame_ids: + original = source.frame(frame_id) + frame = pycolmap.Frame(frame_id=frame_id, rig_id=original.rig_id) + for data in original.image_ids: + frame.add_data_id(data) + rec.add_frame(frame) + for image in images: + rec.add_image( + pycolmap.Image( + image_id=image.image_id, name=image.name, camera_id=image.camera_id, frame_id=image.frame_id + ) + ) + return rec diff --git a/vidmap/datasets/frame_names.py b/vidmap/datasets/frame_names.py index 551fce2..eabf86d 100644 --- a/vidmap/datasets/frame_names.py +++ b/vidmap/datasets/frame_names.py @@ -1,18 +1,38 @@ """Image-name ordering shared by datasets, reconstruction, and evaluation.""" +from decimal import Decimal from pathlib import Path -def timestamp_from_image_name(name: str) -> float: - """Parse a numeric timestamp from supported image-name conventions.""" +def _timestamp_text(name: str) -> str | None: stem = Path(name).stem try: - return float(stem) + float(stem) + return stem except ValueError: pass if "-" in stem: try: - return float(stem.split("-")[0]) + prefix = stem.split("-")[0] + float(prefix) + return prefix except ValueError: pass - return 0.0 + return None + + +def timestamp_from_image_name(name: str) -> float: + """Parse a numeric timestamp from supported image-name conventions.""" + text = _timestamp_text(name) + return 0.0 if text is None else float(text) + + +def timestamp_ns_from_image_name(name: str) -> int: + """Resolve a timestamp in seconds without losing nanosecond precision.""" + text = _timestamp_text(name) + if text is None or not Decimal(text).is_finite(): + raise ValueError(f"Cannot resolve timestamp in {name!r}; set rig.frames to a frames.csv") + timestamp = Decimal(text) * 1_000_000_000 + if timestamp != timestamp.to_integral_value(): + raise ValueError(f"Timestamp in {name!r} is finer than nanoseconds; set rig.frames to a frames.csv") + return int(timestamp) diff --git a/vidmap/datasets/local.py b/vidmap/datasets/local.py index 5c00fdb..8db3700 100644 --- a/vidmap/datasets/local.py +++ b/vidmap/datasets/local.py @@ -1,10 +1,7 @@ -from collections.abc import Mapping, Sequence -from numbers import Real +from collections.abc import Sequence from pathlib import Path -import numpy as np import pycolmap -import yaml from natsort import natsorted from PIL import Image @@ -13,7 +10,7 @@ IMAGE_SUFFIXES = frozenset({".jpg", ".jpeg", ".png"}) -def _select_images(rgb_dir: Path, imnames: Sequence[str] | None) -> list[str]: +def select_images(rgb_dir: Path, imnames: Sequence[str] | None) -> list[str]: if imnames is None: candidates = natsorted( entry.name for entry in rgb_dir.iterdir() if entry.is_file() and entry.suffix.lower() in IMAGE_SUFFIXES @@ -43,86 +40,9 @@ def _select_images(rgb_dir: Path, imnames: Sequence[str] | None) -> list[str]: return validated -def _add_cameras( - rec: pycolmap.Reconstruction, - intrinsics: Mapping, - names: tuple[str, ...], - root: Path, -): - input_names = set(names) - source_keys: dict[int, object] = {} - assigned: dict[str, tuple[object, int]] = {} - - for source_id, camera_data in intrinsics.items(): - if isinstance(source_id, bool) or not isinstance(source_id, (int, str)): - raise ValueError(f"Camera ID must be an integer or decimal string, got {source_id!r}") - digits = source_id[1:] if isinstance(source_id, str) and source_id[:1] in {"+", "-"} else source_id - if isinstance(digits, str) and not digits.isdecimal(): - raise ValueError(f"Camera ID must be an integer or decimal string, got {source_id!r}") - camera_id = int(source_id) - if not 0 <= camera_id < pycolmap.INVALID_CAMERA_ID: - raise ValueError(f"Camera ID must be in [0, {pycolmap.INVALID_CAMERA_ID}), got {source_id!r}") - if camera_id in source_keys: - raise ValueError(f"Camera IDs {source_keys[camera_id]!r} and {source_id!r} both normalize to {camera_id}") - if not isinstance(camera_data, Mapping) or "params" not in camera_data or "images" not in camera_data: - raise ValueError(f"Camera {source_id!r} must define params and images") - - raw_params = camera_data["params"] - if ( - isinstance(raw_params, (str, bytes)) - or not isinstance(raw_params, Sequence) - or any(isinstance(item, bool) or not isinstance(item, Real) for item in raw_params) - ): - raise ValueError(f"Camera {camera_id} PINHOLE params must contain four finite numbers") - params = tuple(float(item) for item in raw_params) - if len(params) != 4 or not np.isfinite(params).all() or params[0] <= 0 or params[1] <= 0: - raise ValueError( - f"Camera {camera_id} PINHOLE params must be [fx, fy, cx, cy] with positive finite focal lengths" - ) - - selected = camera_data["images"] - if selected == "all": - camera_images = names - elif isinstance(selected, list): - camera_images = tuple(selected) - else: - raise ValueError(f"Camera {source_id!r} images must be 'all' or a list") - if not camera_images: - raise ValueError(f"Camera {source_id!r} must select at least one image") - for image_name in camera_images: - if not isinstance(image_name, str) or image_name not in input_names: - raise ValueError(f"Camera {source_id!r} selects unknown input image {image_name!r}") - if image_name in assigned: - raise ValueError( - f"Input image {image_name!r} is assigned to cameras {assigned[image_name][0]!r} and {source_id!r}" - ) - assigned[image_name] = (source_id, camera_id) - - dimensions = [] - for image_name in camera_images: - path = root / image_name - try: - with Image.open(path) as image: - dimensions.append(image.size) - except OSError as error: - raise ValueError(f"Could not read image dimensions from {path}: {error}") from error - if len(set(dimensions)) != 1: - raise ValueError(f"All images assigned to camera {camera_id} must have the same shape") - width, height = dimensions[0] - if width <= 0 or height <= 0: - raise ValueError(f"Camera {camera_id} images must have positive dimensions") - camera = pycolmap.Camera.create_from_model_name(camera_id, "PINHOLE", params[0], width, height) - camera.params = params - camera.has_prior_focal_length = True - rec.add_camera_with_trivial_rig(camera) - source_keys[camera_id] = source_id - - if missing := [image_name for image_name in names if image_name not in assigned]: - raise ValueError(f"Intrinsics do not assign every input image; missing: {missing}") - return {image_name: source[1] for image_name, source in assigned.items()} - - class LocalImageParser(PreparedSceneParser): + """Adapt an uncalibrated local image sequence to native COLMAP frames.""" + scene = "" def __init__( @@ -130,38 +50,30 @@ def __init__( *, image_dir: str | Path, imnames: Sequence[str] | None = None, - intrinsics_path: str | Path | None = None, - estimate_intrinsics: bool = False, ) -> None: self.rgb_dir = Path(image_dir).expanduser() if not self.rgb_dir.is_dir(): raise FileNotFoundError(f"image_dir is not a directory: {self.rgb_dir}") - self.imnames = _select_images(self.rgb_dir, imnames) - - if estimate_intrinsics: - if intrinsics_path is not None: - raise ValueError("intrinsics_path cannot be combined with estimate_intrinsics=True") - intrinsics: Mapping = {1: {"params": [1000.0, 1000.0, 1000.0, 1000.0], "images": "all"}} - else: - if intrinsics_path is None: - raise ValueError("intrinsics_path is required when estimate_intrinsics=False") - path = Path(intrinsics_path).expanduser() - try: - with path.open(encoding="utf-8") as file: - intrinsics = yaml.safe_load(file) - except (UnicodeDecodeError, yaml.YAMLError) as error: - raise ValueError(f"Invalid intrinsics file {path}: {error}") from error - if not isinstance(intrinsics, Mapping) or not intrinsics: - raise ValueError(f"Intrinsics file {path} must define at least one camera mapping") - + self.imnames = select_images(self.rgb_dir, imnames) + + dimensions = set() + for name in self.imnames: + with Image.open(self.rgb_dir / name) as pixels: + dimensions.add(pixels.size) + if len(dimensions) != 1: + raise ValueError("All images in a monocular sequence must have the same shape") + width, height = dimensions.pop() self.rec = pycolmap.Reconstruction() self.reconstruction_dir = None - image_camera_ids = _add_cameras(self.rec, intrinsics, tuple(self.imnames), self.rgb_dir) + camera = pycolmap.Camera.create_from_model_name(1, "PINHOLE", 1000.0, width, height) + camera.params = [1000.0, 1000.0, 1000.0, 1000.0] + camera.has_prior_focal_length = True + self.rec.add_camera_with_trivial_rig(camera) for image_id, image_name in enumerate(self.imnames, start=1): image = pycolmap.Image( name=image_name, - camera_id=image_camera_ids[image_name], + camera_id=camera.camera_id, image_id=image_id, ) self.rec.add_image_with_trivial_frame(image) diff --git a/vidmap/datasets/prepare/filesystem.py b/vidmap/datasets/prepare/filesystem.py index 4939fb1..13bf4da 100644 --- a/vidmap/datasets/prepare/filesystem.py +++ b/vidmap/datasets/prepare/filesystem.py @@ -2,11 +2,15 @@ from __future__ import annotations +import json import shutil import subprocess from collections.abc import Callable, Iterator, Sequence from contextlib import contextmanager from pathlib import Path +from tempfile import TemporaryDirectory + +from vidmap.repro.hashing import sha256_file IMAGE_SUFFIXES = frozenset({".jpg", ".jpeg", ".png"}) _MODEL_FILES = ("cameras.bin", "images.bin", "points3D.bin") @@ -80,3 +84,38 @@ def scene_staging(destination: Path) -> Iterator[Path]: yield temporary finally: shutil.rmtree(temporary, ignore_errors=True) + + +def _prepared_image_hashes(root: Path, names: Sequence[str]) -> dict[str, str]: + paths = {name: root / "images" / name for name in names} + if any(not path.is_file() for path in paths.values()): + raise ValueError("Missing prepared image") + return {name: sha256_file(path) for name, path in paths.items()} + + +def prepare_image_directory( + output: Path, identity: dict, names: Sequence[str], write_images: Callable[[Path], None] +) -> dict[str, str]: + """Write or verify a prepared image directory and return its current hashes.""" + identity = json.loads(json.dumps(identity, sort_keys=True)) + if output.exists() or output.is_symlink(): + manifest = json.loads((output / "manifest.json").read_text()) + if manifest["identity"] != identity: + raise ValueError("Prepared output identity mismatch; refusing overwrite") + hashes = _prepared_image_hashes(output, names) + if manifest != dict(identity=identity, images=hashes): + raise ValueError("Prepared output manifest mismatch; images or fields changed") + return hashes + output.parent.mkdir(parents=True, exist_ok=True) + with TemporaryDirectory(prefix=f".{output.name}-", dir=output.parent) as temporary: + staging = Path(temporary) / "capture" + staging.mkdir() + write_images(staging / "images") + hashes = _prepared_image_hashes(staging, names) + (staging / "manifest.json").write_text( + json.dumps(dict(identity=identity, images=hashes), sort_keys=True, indent=2) + "\n" + ) + if output.exists() or output.is_symlink(): + raise FileExistsError(f"Output appeared while preparing: {output}") + staging.rename(output) + return hashes diff --git a/vidmap/datasets/prepare/rig_capture.py b/vidmap/datasets/prepare/rig_capture.py new file mode 100644 index 0000000..169f493 --- /dev/null +++ b/vidmap/datasets/prepare/rig_capture.py @@ -0,0 +1,127 @@ +"""Load calibrated physical rigs and prepare images with COLMAP undistortion.""" + +from dataclasses import replace +from pathlib import Path + +import numpy as np +from PIL import Image + +from vidmap.datasets.capture import ( + CaptureSnapshot, + PreparedCapture, + build_capture_reconstruction, + resolve_capture_frames, +) +from vidmap.datasets.prepare.filesystem import prepare_image_directory + + +def load_capture(image_dir: str | Path, options) -> PreparedCapture: + """Load inline calibration and synchronized images from cam0/, cam1/, ... .""" + import pycolmap + + root = Path(image_dir).expanduser().resolve(strict=True) + if not root.is_dir(): + raise ValueError("Capture images must be a directory") + if not options.cameras: + raise ValueError("Physical rig input requires rig.cameras") + rows = resolve_capture_frames(root, tuple(f"cam{cid}" for cid in range(len(options.cameras))), options.frames) + sizes = {} + for row in rows: + camera_id, name = row["camera_id"], row["image_name"] + if camera_id >= len(options.cameras) or not name.startswith(f"cam{camera_id}/"): + raise ValueError(f"Image {name!r} must belong to a declared camera's camN/ directory") + if camera_id not in sizes: + with Image.open(root / name) as pixels: + sizes[camera_id] = pixels.size + if len(sizes) != len(options.cameras): + raise ValueError("Every declared camera requires images") + cameras = [] + rig = pycolmap.Rig(rig_id=1) + for camera_id, source in enumerate(options.cameras): + width, height = sizes[camera_id] + camera = pycolmap.Camera( + camera_id=camera_id, + model=pycolmap.CameraModelId(source.camera_model_name), + width=width, + height=height, + params=source.camera_params, + has_prior_focal_length=True, + ) + if not camera.verify_params(): + raise ValueError(f"Invalid parameter count for camera model {camera.model_name}") + if any(camera.params[index] <= 0 for index in camera.focal_length_idxs()): + raise ValueError("Camera focal lengths must be positive") + cameras.append(camera) + rig.add_ref_sensor(cameras[0].sensor_id) + for camera, source in zip(cameras[1:], options.cameras[1:], strict=True): + # YAML uses wxyz; Rotation3d's quaternion constructor takes xyzw. + w, x, y, z = source.cam_from_rig_rotation + rig.add_sensor( + camera.sensor_id, + pycolmap.Rigid3d(pycolmap.Rotation3d([x, y, z, w]), source.cam_from_rig_translation), + ) + rec, frame_times = build_capture_reconstruction(cameras, rig, rows) + return PreparedCapture( + rec=rec, + rgb_dir=root, + frame_timestamps_ns=frame_times, + kind="rig", + provenance={}, + ) + + +def prepare_rig_capture( + capture: PreparedCapture, output_dir: str | Path, *, camera_ids_to_undistort: tuple[int, ...] +) -> CaptureSnapshot: + """Undistort physical rigs once; pinhole-only captures keep their source images.""" + import pycolmap + + distorted = set(camera_ids_to_undistort) + if not distorted: + return capture.snapshot() + + source = capture.snapshot() + options = pycolmap.UndistortCameraOptions() + reconstruction = pycolmap.Reconstruction(capture.rec) + for camera_id in distorted: + original = capture.rec.camera(camera_id) + camera = pycolmap.undistort_camera(options, original) + camera.camera_id = camera_id + camera.has_prior_focal_length = original.has_prior_focal_length + reconstruction.cameras[camera_id] = camera + # Appending a suffix preserves distinct source names and writes lossless pixels. + for image in reconstruction.images.values(): + image.name += ".png" + + output = Path(output_dir).expanduser().absolute() + identity = dict( + schema_version=2, + colmap_version=pycolmap.__version__, + source_fingerprint=source.fingerprint(), + undistort_camera_ids=sorted(distorted), + ) + names = tuple(image.name for image in reconstruction.images.values()) + + def write_images(image_dir): + for image_id, image in capture.rec.images.items(): + with Image.open(capture.rgb_dir / image.name) as pixels: + array = np.asarray(pixels.convert("RGB")) + if image.camera_id in distorted: + bitmap, _ = pycolmap.undistort_image( + options, pycolmap.Bitmap.from_array(array), capture.rec.camera(image.camera_id) + ) + array = bitmap.to_array() + target = image_dir / reconstruction.image(image_id).name + target.parent.mkdir(parents=True, exist_ok=True) + Image.fromarray(array).save(target) + if capture.snapshot().fingerprint() != identity["source_fingerprint"]: + raise ValueError("Source capture changed while undistorting") + + hashes = prepare_image_directory(output, identity, names, write_images) + prepared = replace( + capture, + rec=reconstruction, + rgb_dir=output / "images", + provenance={"source_fingerprint": identity["source_fingerprint"], "image_sha256": hashes}, + ) + return CaptureSnapshot(prepared, hashes) diff --git a/vidmap/frontend/cache/validation.py b/vidmap/frontend/cache/validation.py index fec388d..b2587d4 100644 --- a/vidmap/frontend/cache/validation.py +++ b/vidmap/frontend/cache/validation.py @@ -298,7 +298,7 @@ def validate_complete_payload(path: Path, metadata: Mapping[str, Any]) -> None: if set(hfile.keys()) == {PAYLOAD_DATASET} and isinstance(hfile[PAYLOAD_DATASET], h5py.Dataset): data = hfile[PAYLOAD_DATASET][:] stage = metadata.get("stage") - if stage in {"track_pairs", "retrieval_pairs"}: + if stage == "retrieval_pairs": if data.ndim != 2 or data.shape[1] != 2 or data.dtype.kind not in {"O", "S", "U"}: raise CacheMetadataMismatch(f"{path}: malformed pair payload") decode_pair_array(path, data) diff --git a/vidmap/frontend/cli.py b/vidmap/frontend/cli.py index 2d6e614..866417d 100644 --- a/vidmap/frontend/cli.py +++ b/vidmap/frontend/cli.py @@ -19,7 +19,6 @@ def build_parser() -> argparse.ArgumentParser: help="Project directory for decoded frames, frontend artifacts, and mapper inputs.", ) parser.add_argument("--imnames", nargs="*", type=str) - parser.add_argument("--intrinsics", type=str) parser.add_argument( "--mapper-inputs", help="Finalized mapper inputs to use when backfilling full depth maps into --output.", @@ -78,7 +77,6 @@ def main(argv=None): args.input, workspace=args.output, imnames=args.imnames, - intrinsics_path=args.intrinsics, force_frontend=args.force_frontend, cache_depth_maps=args.cache_depth_maps, mapper_inputs_path=args.mapper_inputs, diff --git a/vidmap/frontend/colmap_database.py b/vidmap/frontend/colmap_database.py index afe36b5..9818eeb 100644 --- a/vidmap/frontend/colmap_database.py +++ b/vidmap/frontend/colmap_database.py @@ -9,21 +9,22 @@ from contextlib import contextmanager, redirect_stdout from io import StringIO from pathlib import Path -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING import numpy as np import pycolmap from tqdm import tqdm -from vidmap.frontend.cache import CacheMetadataMismatch, read_pair_artifact, validate_incremental_cache +from vidmap.datasets.capture import copy_unposed_reconstruction +from vidmap.frontend.cache import validate_incremental_cache from vidmap.frontend.correspondences import ImagePair +from vidmap.mapper.inputs.snapshot import require_finalized_sqlite from vidmap.utils.io import get_keypoints, get_matches from vidmap.utils.logging import progress_bars_enabled if TYPE_CHECKING: from vidmap.frontend.pipeline import TrackingFrontendResult -CameraPolicy = Literal["transitive", "per_image", "shared"] logger = logging.getLogger(__name__) @@ -99,22 +100,39 @@ def import_features(image_ids: Mapping[str, int], database_path: Path, features_ db.close() +def _transitive_reconstruction(reconstruction, image_names): + """Give each diagnostic image a fresh singleton camera, in name order.""" + names = set(image_names) + images = sorted( + (image for image in reconstruction.images.values() if image.name in names), key=lambda image: image.name + ) + if {image.name for image in images} != names: + raise ValueError("Database reconstruction is missing requested images") + selected = pycolmap.Reconstruction() + for camera_id, image in enumerate(images, start=1): + camera = pycolmap.Camera(reconstruction.camera(image.camera_id).todict()) + camera.camera_id = camera_id + selected.add_camera_with_trivial_rig(camera) + selected.add_image_with_trivial_frame( + pycolmap.Image(name=image.name, camera_id=camera_id, image_id=image.image_id) + ) + return selected + + def build_colmap_database( database_path: Path, - reconstruction, + reconstruction: pycolmap.Reconstruction, image_names: Sequence[str], sparse_features_path: Path, pairs: Sequence[ImagePair], *, - camera_policy: CameraPolicy, prior_focal_length: bool, + diagnostic_transitive: bool = False, sparse_matches_path: Path | None = None, matches: Mapping[ImagePair, object] | None = None, seed_two_view_geometry: bool = False, -) -> dict[str, int]: - """Create and populate one frontend-owned COLMAP database.""" - if camera_policy not in ("transitive", "per_image", "shared"): - raise ValueError(f"Unsupported camera policy: {camera_policy}") +) -> pycolmap.Reconstruction: + """Serialize prepared topology, or explicitly flatten a diagnostic database.""" if (sparse_matches_path is None) == (matches is None): raise ValueError("Exactly one match source is required") image_names = tuple(image_names) @@ -123,85 +141,32 @@ def build_colmap_database( if len(set(image_names)) != len(image_names): raise ValueError("Database image names must be unique") - database_path.unlink(missing_ok=True) + require_finalized_sqlite(database_path) build_path = database_path.with_name(f".{database_path.name}.building") build_path.unlink(missing_ok=True) - db = pycolmap.Database.open(build_path) - base_succeeded = False + logger.info("Building COLMAP database at %s", build_path) try: - with pycolmap.DatabaseTransaction(db): - selected_names = set(image_names) - image_ids: dict[str, int] = {} - shared_camera_added = False - shared_camera_signature = None - transitive_camera_id = 1 - - for reconstruction_index, (image_id, image) in enumerate( - sorted(reconstruction.images.items(), key=lambda item: item[1].name) - ): - imname = image.name - if imname not in selected_names: - continue - camera = reconstruction.cameras[image.camera_id] - - image_ids[imname] = image_id - - write_camera = True - if camera_policy == "shared": - database_camera_id = 0 - signature = ( - camera.model, - camera.width, - camera.height, - tuple(np.asarray(camera.params).tolist()), - ) - if shared_camera_signature is None: - shared_camera_signature = signature - elif signature != shared_camera_signature: - raise ValueError( - "Shared-camera database policy requires identical camera models, dimensions, and parameters" - ) - if shared_camera_added: - write_camera = False - else: - shared_camera_added = True + with pycolmap.Database.open(build_path) as db: + with pycolmap.DatabaseTransaction(db): + if diagnostic_transitive: + selected = _transitive_reconstruction(reconstruction, image_names) else: - if camera_policy == "transitive": - database_camera_id = transitive_camera_id - transitive_camera_id += 1 - else: - database_camera_id = reconstruction_index - - if write_camera: - db.write_camera( - pycolmap.Camera( - camera_id=database_camera_id, - model=camera.model, - width=camera.width, - height=camera.height, - params=camera.params, - has_prior_focal_length=prior_focal_length, - ), - use_camera_id=True, - ) - db.write_image( - pycolmap.Image(name=imname, camera_id=database_camera_id, image_id=image_id), - use_image_id=True, - ) - - missing_names = sorted(selected_names - set(image_ids)) - if missing_names: - raise ValueError(f"Database reconstruction is missing requested image {missing_names[0]!r}") - base_succeeded = True - finally: - try: - db.close() - finally: - if not base_succeeded: - build_path.unlink(missing_ok=True) - - imports_succeeded = False - try: + selected = copy_unposed_reconstruction(reconstruction, image_names) + for camera in selected.cameras.values(): + camera.has_prior_focal_length = prior_focal_length + images = selected.images.values() + if diagnostic_transitive: + images = sorted(images, key=lambda image: image.name) + image_ids = {image.name: image.image_id for image in images} + camera_ids = dict.fromkeys(selected.image(image_id).camera_id for image_id in image_ids.values()) + for camera_id in camera_ids: + db.write_camera(selected.camera(camera_id), use_camera_id=True) + for rig in selected.rigs.values(): + db.write_rig(rig, use_rig_id=True) + for frame in selected.frames.values(): + db.write_frame(frame, use_frame_id=True) + for image_id in image_ids.values(): + db.write_image(selected.image(image_id), use_image_id=True) import_features(image_ids, build_path, sparse_features_path) _import_matches( image_ids, @@ -211,12 +176,10 @@ def build_colmap_database( matches=matches, seed_two_view_geometry=seed_two_view_geometry, ) - imports_succeeded = True + build_path.replace(database_path) finally: - if not imports_succeeded: - build_path.unlink(missing_ok=True) - build_path.replace(database_path) - return image_ids + build_path.unlink(missing_ok=True) + return selected def create_database_from_frontend( @@ -225,14 +188,11 @@ def create_database_from_frontend( database_path: Path, *, estimate_intrinsics: bool, - matches: dict[ImagePair, object] | None = None, -) -> dict[str, int]: + matches: Mapping[ImagePair, object], +) -> pycolmap.Reconstruction: """Build a COLMAP database from a validated frontend result.""" paths = frontend_result.paths artifacts = frontend_result.artifacts - pairs = read_pair_artifact(paths.track_pairs_path, artifacts.track_pairs.metadata) - if tuple(pairs) != tuple(frontend_result.track_pairs): - raise CacheMetadataMismatch("Track-pair cache changed after frontend") validate_incremental_cache( paths.sparse_features_path, artifacts.sparse_features.metadata, @@ -243,21 +203,13 @@ def create_database_from_frontend( artifacts.sparse_matches.metadata, artifacts.sparse_matches.expected_items, ) - - import_pairs = ( - tuple(frontend_result.track_pairs) - if matches is None - else tuple(sorted(matches, key=lambda pair: (str(pair[0]), str(pair[1])))) - ) return build_colmap_database( database_path, initial_reconstruction, frontend_result.keyframe_sequence, paths.sparse_features_path, - import_pairs, - camera_policy="shared" if estimate_intrinsics else "per_image", + tuple(sorted(matches, key=lambda pair: (str(pair[0]), str(pair[1])))), prior_focal_length=not estimate_intrinsics, - sparse_matches_path=paths.sparse_matches_path if matches is None else None, matches=matches, ) diff --git a/vidmap/frontend/depth.py b/vidmap/frontend/depth.py index cd06dd5..2d1cec2 100644 --- a/vidmap/frontend/depth.py +++ b/vidmap/frontend/depth.py @@ -14,6 +14,7 @@ from tqdm import tqdm from vidmap.datasets.base import DatasetParser +from vidmap.datasets.capture import PreparedCapture, prepare_scene_capture from vidmap.frontend.cache import ( IncrementalArtifactContract, artifact_fingerprint, @@ -36,6 +37,7 @@ ) from vidmap.frontend.options.depth import Da3VideoOptions, DepthEstimationOptions from vidmap.frontend.paths import FrontendPaths +from vidmap.frontend.rig import camera_streams from vidmap.utils.device import empty_device_cache from vidmap.utils.image_sampling import sample_at_keypoints from vidmap.utils.logging import progress_bars_enabled @@ -79,6 +81,13 @@ def _depth_options_cache_config(options): return semantic_config(options, exclude_fields=_RUNTIME_CONFIG_FIELDS) +def _depth_ordered_inputs(image_names, image_content_fingerprint, stream_sequences): + inputs = {"images": image_names, "image_content": image_content_fingerprint} + if stream_sequences != (tuple(image_names),): + inputs["stream_sequences"] = stream_sequences + return inputs + + def _prepare_full_depth_cache( path: Path, *, @@ -86,6 +95,7 @@ def _prepare_full_depth_cache( image_content_fingerprint: str, options: DepthEstimationOptions, overwrite: bool, + stream_sequences, ): """Prepare one resumable confidence-free full-depth cache.""" metadata = cache_metadata( @@ -95,10 +105,7 @@ def _prepare_full_depth_cache( "backend": da3_cache_identity(options.backend), "native_resolution": True, }, - ordered_inputs={ - "images": image_names, - "image_content": image_content_fingerprint, - }, + ordered_inputs=_depth_ordered_inputs(image_names, image_content_fingerprint, stream_sequences), payload_format="per-image-full-depth-v2", nonsemantic_config_fields=_RUNTIME_CONFIG_FIELDS, ) @@ -230,6 +237,7 @@ def _run_da3_depth( full_names=frozenset(), *, num_workers, + stream_sequences, calibration_enabled=False, batch_size=1, device: torch.device, @@ -254,6 +262,7 @@ def _run_da3_depth( process_res=backend.process_res, num_workers=num_workers, pin_memory=(device.type == "cuda"), + stream_sequences=stream_sequences, ) windows = iter(loader) model = None @@ -342,7 +351,7 @@ class DepthEstimator: def __init__( self, *, - scene_parser: DatasetParser, + scene_parser: PreparedCapture, paths: FrontendPaths, force_recompute: bool, keyframes: KeyframePlan, @@ -367,16 +376,14 @@ def estimate(self, *, cache_full_depth_maps: bool = False) -> DepthFrontendResul logger.info("Estimating depth maps for %d keyframes", len(image_names)) started = sync_time() sparse_features_metadata = read_cache_metadata(self.paths.sparse_features_path) + stream_sequences = self.keyframes.streams metadata = cache_metadata( stage="depth", config={ "options": _depth_options_cache_config(self.options), "backend": da3_cache_identity(self.options.backend), }, - ordered_inputs={ - "images": image_names, - "image_content": self.image_content_fingerprint, - }, + ordered_inputs=_depth_ordered_inputs(image_names, self.image_content_fingerprint, stream_sequences), upstream={"sparse_features": artifact_fingerprint(sparse_features_metadata)}, payload_format=( "per-image-keypoint-depth-calibration" if self.calibration_enabled else "per-image-keypoint-depth" @@ -412,6 +419,7 @@ def estimate(self, *, cache_full_depth_maps: bool = False) -> DepthFrontendResul image_content_fingerprint=self.image_content_fingerprint, options=self.options, overwrite=self.force_recompute, + stream_sequences=stream_sequences, ) sampled_names = frozenset(sampled_pending_names) full_names = frozenset(full_pending_names) @@ -429,6 +437,7 @@ def estimate(self, *, cache_full_depth_maps: bool = False) -> DepthFrontendResul full_depth_path if cache_full_depth_maps else None, full_names, num_workers=self.options.num_workers, + stream_sequences=stream_sequences, calibration_enabled=self.calibration_enabled, batch_size=self.options.batch_size, device=self.device, @@ -451,7 +460,7 @@ def estimate(self, *, cache_full_depth_maps: bool = False) -> DepthFrontendResul def cache_full_depth_maps_posthoc( *, - scene_parser: DatasetParser, + scene_parser: DatasetParser | PreparedCapture, image_names, sampled_depth_path: Path, output_path: Path, @@ -462,16 +471,19 @@ def cache_full_depth_maps_posthoc( """Infer only full prediction-grid depths for an existing frontend.""" from vidmap.frontend.cache import ordered_files_fingerprint + scene_parser = prepare_scene_capture(scene_parser) image_names = tuple(image_names) if not image_names: raise ValueError("Post-hoc full-depth frontend requires at least one finalized keyframe") output_path = Path(output_path) + stream_sequences = camera_streams(scene_parser, image_names) metadata, missing = _prepare_full_depth_cache( output_path, image_names=image_names, image_content_fingerprint=ordered_files_fingerprint(scene_parser.rgb_dir, image_names), options=options, overwrite=force_recompute, + stream_sequences=stream_sequences, ) if missing: logger.info("Post-hoc full-depth frontend: processing %d keyframes", len(missing)) @@ -486,6 +498,7 @@ def cache_full_depth_maps_posthoc( output_path, frozenset(missing), num_workers=options.num_workers, + stream_sequences=stream_sequences, batch_size=options.batch_size, device=device, ) diff --git a/vidmap/frontend/depth_loading.py b/vidmap/frontend/depth_loading.py index f6c39ba..cfad6be 100644 --- a/vidmap/frontend/depth_loading.py +++ b/vidmap/frontend/depth_loading.py @@ -79,28 +79,37 @@ def __getitem__(self, index: int): class WindowDataset(Dataset): """Load one explicit image window for each pending center image.""" - def __init__(self, image_dataset: ImageDataset, pending_names, window_size: int): + def __init__( + self, + image_dataset: ImageDataset, + pending_names, + window_size: int, + stream_sequences, + ): self.image_dataset = image_dataset self.pending_names = tuple(pending_names) self.window_size = window_size self.image_to_index = {name: index for index, name in enumerate(self.image_dataset.names)} + self.stream_positions = { + name: (stream, index) for stream in stream_sequences for index, name in enumerate(stream) + } def __len__(self) -> int: return len(self.pending_names) def __getitem__(self, item: int): name = self.pending_names[item] - index = self.image_to_index[name] + stream, index = self.stream_positions[name] half_window = self.window_size // 2 start = max(0, index - half_window) - end = min(len(self.image_dataset), index + half_window + 1) + end = min(len(stream), index + half_window + 1) if end - start < self.window_size: if start == 0: - end = min(len(self.image_dataset), self.window_size) + end = min(len(stream), self.window_size) else: - start = max(0, len(self.image_dataset) - self.window_size) + start = max(0, len(stream) - self.window_size) - items = [self.image_dataset[image_index] for image_index in range(start, end)] + items = [self.image_dataset[self.image_to_index[stream[image_index]]] for image_index in range(start, end)] center_index = index - start return { "name": name, @@ -125,10 +134,11 @@ def create_da3_window_loader( process_res: int, num_workers: int, pin_memory: bool = True, + stream_sequences, ) -> DataLoader: """Create the ordered worker pool that prepares DA3 input windows.""" image_dataset = _Da3ImageDataset(rgb_dir, image_names, process_res) - dataset = WindowDataset(image_dataset, pending_names, window_size) + dataset = WindowDataset(image_dataset, pending_names, window_size, stream_sequences) return DataLoader( dataset, batch_size=1, diff --git a/vidmap/frontend/identity.py b/vidmap/frontend/identity.py index 72c578c..2a0c8b8 100644 --- a/vidmap/frontend/identity.py +++ b/vidmap/frontend/identity.py @@ -9,7 +9,7 @@ from vidmap.configuration.names import config_name_to_output_slug from vidmap.frontend.cache import fingerprint -FRONTEND_IDENTITY_SCHEMA_VERSION = 4 +FRONTEND_IDENTITY_SCHEMA_VERSION = 6 def semantic_frontend_config(conf) -> dict[str, object]: @@ -37,11 +37,6 @@ def frontend_config_identity(conf) -> dict[str, object]: "config_name": conf.name, "config_fingerprint": semantic_frontend_fingerprint(conf), "colmap_runtime": conf.colmap_runtime, - "boundary_options": { - "estimator": conf.pipeline.camera_priors.estimator, - "inference": conf.pipeline.camera_priors.inference, - "initialization": conf.pipeline.camera_priors.initialization, - }, } @@ -58,9 +53,7 @@ class FrontendIdentity: mode: str testset_id: str reference_image_ids: tuple[int, ...] - estimator: str - inference: str - initialization: str + source_fingerprint: str | None = None @classmethod def from_config( @@ -72,6 +65,7 @@ def from_config( mode: str, testset_id: str, reference_image_ids: Sequence[int], + source_fingerprint: str | None = None, ) -> "FrontendIdentity": if conf.name is None: raise ValueError("FrontendConfig requires its canonical root-relative name") @@ -86,13 +80,11 @@ def from_config( mode=mode, testset_id=testset_id, reference_image_ids=tuple(reference_image_ids), - estimator=conf.pipeline.camera_priors.estimator, - inference=conf.pipeline.camera_priors.inference, - initialization=conf.pipeline.camera_priors.initialization, + source_fingerprint=source_fingerprint, ) def as_dict(self) -> dict[str, object]: - return { + payload = { "schema_version": FRONTEND_IDENTITY_SCHEMA_VERSION, "tag": self.tag, "config_name": self.config_name, @@ -103,9 +95,10 @@ def as_dict(self) -> dict[str, object]: "mode": self.mode, "testset_id": self.testset_id, "reference_image_ids": list(self.reference_image_ids), - "boundary_options": { - "estimator": self.estimator, - "inference": self.inference, - "initialization": self.initialization, - }, } + + # CPU-only lookup identities select an existing boundary without reopening media. + # Every published boundary requires this field in the manifest validator. + if self.source_fingerprint is not None: + payload["source_fingerprint"] = self.source_fingerprint + return payload diff --git a/vidmap/frontend/image_dataset.py b/vidmap/frontend/image_dataset.py index 2f760ba..71d1d23 100644 --- a/vidmap/frontend/image_dataset.py +++ b/vidmap/frontend/image_dataset.py @@ -1,10 +1,12 @@ +# Strict integer checks intentionally reject bool. +# ruff: noqa: E721 import collections.abc as collections import glob from dataclasses import dataclass from dataclasses import field as dc_field +from functools import cached_property from pathlib import Path -from types import MappingProxyType -from typing import Any, Mapping, Optional +from typing import Optional import cv2 import numpy as np @@ -14,46 +16,60 @@ from torch.nn import functional as F from vidmap.configuration.validators import dataclass as pydantic_dataclass -from vidmap.datasets.frame_names import timestamp_from_image_name +from vidmap.frontend.options.rig import RigOptions +from vidmap.frontend.rig import camera_streams, resolve_rig_options from vidmap.utils.io import read_image from vidmap.utils.parsers import parse_image_lists @dataclass(frozen=True) class FrameSequence: - """Ordered video images with timestamps resolved once per frontend run.""" - - names: tuple[str, ...] - timestamps: Mapping[str, Any] - - def __post_init__(self): - names = tuple(self.names) - missing = [name for name in names if name not in self.timestamps] - if missing: - raise ValueError(f"Missing timestamps for {len(missing)} frontend frames") - object.__setattr__(self, "names", names) - object.__setattr__( - self, - "timestamps", - MappingProxyType({name: self.timestamps[name] for name in names}), - ) + """Camera-local execution streams, with the admission camera first.""" + + streams: tuple[tuple[str, ...], ...] + same_frame_stream_pairs: tuple[tuple[int, int, bool], ...] = () + frame_ids: tuple[int, ...] = dc_field(kw_only=True) + stream_camera_ids: tuple[int, ...] = dc_field(kw_only=True) + + @cached_property + def names(self) -> tuple[str, ...]: + return tuple(name for stream in self.streams for name in stream) + + def select(self, indices): + """Apply admission-camera indices to every synchronized stream.""" + indices = tuple(indices) + return tuple(tuple(stream[i] for i in indices) for stream in self.streams) + + def indices(self, frame_ids): + """Resolve an ordered native frame selection against this timeline.""" + frame_ids = tuple(frame_ids) + if any(type(frame_id) is not int for frame_id in frame_ids): + raise ValueError("Selected native frame IDs must be integers") + positions = {frame_id: i for i, frame_id in enumerate(self.frame_ids)} + if any(frame_id not in positions for frame_id in frame_ids): + raise ValueError("Selected frame is absent from the capture timeline") + indices = tuple(positions[frame_id] for frame_id in frame_ids) + if any(b <= a for a, b in zip(indices, indices[1:])): + raise ValueError("Selected frames must be unique and follow the capture timeline") + return indices @classmethod - def from_scene(cls, scene_parser): - names, timestamps = build_sequence_and_timestamps(scene_parser) - return cls(tuple(names), timestamps) - - -def build_sequence_and_timestamps(scene_parser): - """Build the ordered image sequence and its timestamps.""" - reconstruction = scene_parser.vio_rec if hasattr(scene_parser, "vio_rec") else scene_parser.rec - ids = sorted(reconstruction.images) - reconstruction_ids = list(scene_parser.rec.images) - minimum_id = min(reconstruction_ids) - maximum_id = max(reconstruction_ids) - sequence = [reconstruction.images[image_id].name for image_id in ids if minimum_id <= image_id <= maximum_id] - sequence = sorted(sequence, key=timestamp_from_image_name) - return sequence, {name: timestamp_from_image_name(name) for name in sequence} + def from_scene(cls, scene_parser, *, rig_options: RigOptions | None = None): + options = RigOptions() if rig_options is None else rig_options + reference, policy = resolve_rig_options(scene_parser, options) + streams = camera_streams(scene_parser, reference_camera_id=reference) + same_frame_stream_pairs = () + if policy: + positions = {scene_parser.image(stream[0]).camera_id: index for index, stream in enumerate(streams)} + same_frame_stream_pairs = tuple( + (positions[first], positions[second], sequential) for first, second, sequential in policy + ) + return cls( + streams, + same_frame_stream_pairs, + frame_ids=tuple(scene_parser.image(name).frame_id for name in streams[0]), + stream_camera_ids=tuple(scene_parser.image(stream[0]).camera_id for stream in streams), + ) def get_image_size(scene_parser, image_name): diff --git a/vidmap/frontend/initial_reconstruction.py b/vidmap/frontend/initial_reconstruction.py deleted file mode 100644 index 0662fa3..0000000 --- a/vidmap/frontend/initial_reconstruction.py +++ /dev/null @@ -1,20 +0,0 @@ -from copy import deepcopy - -import pycolmap - - -def build_initial_reconstruction( - scene_parser, reference_image_names: list[str] | None = None -) -> pycolmap.Reconstruction: - """Copy selected images and their cameras from the prepared scene.""" - source = scene_parser.rec - names = None if reference_image_names is None else set(reference_image_names) - images = [image for image in source.images.values() if names is None or image.name in names] - rec = pycolmap.Reconstruction() - for camera_id in dict.fromkeys(image.camera_id for image in images): - rec.add_camera_with_trivial_rig(deepcopy(source.cameras[camera_id])) - for image in images: - rec.add_image_with_trivial_frame( - pycolmap.Image(image_id=image.image_id, name=image.name, camera_id=image.camera_id) - ) - return rec diff --git a/vidmap/frontend/keyframes/cache.py b/vidmap/frontend/keyframes/cache.py index 1a49953..4b4c43f 100644 --- a/vidmap/frontend/keyframes/cache.py +++ b/vidmap/frontend/keyframes/cache.py @@ -1,18 +1,16 @@ -"""Admitted track-pair and salient-feature cache identities and publication.""" +"""Native frame selection and salient-feature cache publication.""" import logging +import numpy as np + from vidmap.frontend.cache import ( + CacheMetadataMismatch, cache_is_valid, cache_metadata, - incremental_cache_is_complete, - mark_incremental_cache_complete, - ordered_files_fingerprint, - prune_incremental_items, - read_cache_metadata, - read_pair_artifact, + read_single_dataset, semantic_config, - write_pair_artifact, + write_single_dataset, ) from vidmap.frontend.geocalib import keyframe_bootstrap_cache_identity from vidmap.frontend.keyframes import selector as keyframe_selector @@ -24,186 +22,118 @@ logger = logging.getLogger("vidmap.frontend.keyframes.processing") -def _ordered_timestamps(sequence, timestamps): - values = [] - for name in sequence: - value = timestamps[name] - if hasattr(value, "isoformat"): - value = value.isoformat() - values.append([name, value]) - return values - - -def admitted_track_pairs_cache_metadata( +def admitted_frames_cache_metadata( *, - scene_parser, + input_snapshot, sequence, - timestamps, + frame_ids, tracker_options, + track_options, lowres_options, highres_options, keyframe_options, salient_options, ): - """Build the admitted adjacent-pair identity from both selection passes.""" + """Bind admission to the reference stream and its native frame timeline.""" + scene_parser = input_snapshot.capture + image_fingerprint = input_snapshot.image_content_fingerprint intrinsics_source = keyframe_options.intrinsics_source keyframe_config = semantic_config(keyframe_options, exclude_fields=frozenset({"intrinsics_source"})) config = { "tracker": romav2_cache_identity(tracker_options), "lowres": lowres_options, - "highres": highres_options, "keyframes": keyframe_config, "salient_features": semantic_config(salient_options), "salient_feature_model": aliked_cache_identity(), } + if keyframe_options.lookahead_pruning: + config["lookahead_tracks"] = semantic_config(track_options) + config["highres"] = highres_options if lowres_options.batch_size > 1: # Zero-pad the final partial batch to reuse the full-batch compiled graph. config["lowres_batch_padding"] = "zeros-to-configured-batch-size" ordered_inputs = { "sequence": sequence, - "timestamps": _ordered_timestamps(sequence, timestamps), - "image_content": ordered_files_fingerprint(scene_parser.rgb_dir, sequence), + "frame_ids": frame_ids, + "image_content": image_fingerprint(sequence), } + if keyframe_options.force_gt_keyframes: + ordered_inputs["forced_keyframe_names"] = [ + name for name in sequence if name in scene_parser.forced_keyframe_names + ] if intrinsics_source == "geocalib": bootstrap_config, bootstrap_inputs = keyframe_bootstrap_cache_identity(sequence) config["bootstrap_geocalib"] = bootstrap_config ordered_inputs["bootstrap"] = bootstrap_inputs - ordered_inputs["bootstrap_image_content"] = ordered_files_fingerprint( - scene_parser.rgb_dir, - bootstrap_inputs["images"], - ) + ordered_inputs["bootstrap_image_content"] = image_fingerprint(bootstrap_inputs["images"]) else: config["ground_truth_intrinsics"] = {"policy": "ordered-effective-reconstruction-calibration-v1"} ordered_inputs["ground_truth_intrinsics"] = keyframe_selector.ground_truth_intrinsics_plan( scene_parser, sequence ) return cache_metadata( - stage="track_pairs", + stage="admitted_frames", config=config, ordered_inputs=ordered_inputs, - payload_format="ordered-image-pairs", + payload_format="ordered-native-frame-ids", nonsemantic_config_fields=_RUNTIME_CONFIG_FIELDS, ) -def salient_feature_cache_metadata(*, salient_options, sequence, timestamps, track_pairs_metadata): - """Build the salient-feature identity from the admitted pair plan.""" - track_pairs_fingerprint = track_pairs_metadata.get( - "artifact_fingerprint", - track_pairs_metadata["identity_fingerprint"], - ) +def salient_feature_cache_metadata(*, salient_options): + """Feature inference depends on its model; source bytes are bound per image.""" return cache_metadata( stage="salient_features", config={ "features": semantic_config(salient_options), "model": aliked_cache_identity(), }, - ordered_inputs={ - "sequence": sequence, - "timestamps": _ordered_timestamps(sequence, timestamps), - }, - upstream={"track_pairs": track_pairs_fingerprint}, + ordered_inputs={}, payload_format="per-image-local-features", nonsemantic_config_fields=_RUNTIME_CONFIG_FIELDS, ) -def load_cached_names( - *, - scene_parser, - sequence, - timestamps, - track_pairs_path, - salient_features_path, - force_recompute, - keyframe_options, - salient_options, - track_pairs_metadata, +def load_cached_frame_ids( + *, frames, scene_parser, keyframe_options, admitted_frames_path, force_recompute, admission_metadata ): - if force_recompute or not cache_is_valid( - track_pairs_path, - track_pairs_metadata, - ): + """Read the selected native frame IDs; image streams are a topology projection.""" + if force_recompute or not cache_is_valid(admitted_frames_path, admission_metadata): return None - pairs = tuple(read_pair_artifact(track_pairs_path, track_pairs_metadata)) - if not pairs: - logger.info("Cached admitted track-pair plan is empty") - return None - names = (pairs[0][0], *(pair[1] for pair in pairs)) - if pairs != tuple(zip(names, names[1:])): - logger.info("Cached admitted track pairs do not form one adjacent chain") - return None - positions = {name: index for index, name in enumerate(sequence)} - if any(name not in positions for name in names): - logger.info("Cached admitted track pairs contain images outside the source sequence") - return None - keyframe_ids = [positions[name] for name in names] - last_frame_idx = len(sequence) - 1 - if ( - keyframe_ids[0] != 0 - or keyframe_ids[-1] != last_frame_idx - or any(right <= left for left, right in zip(keyframe_ids, keyframe_ids[1:])) - ): - logger.info( - f"Cached keyframes invalid: first={keyframe_ids[0] if keyframe_ids else None}, " - f"last={keyframe_ids[-1] if keyframe_ids else None}, expected first=0, last={last_frame_idx}" - ) - return None - logger.debug("Loaded admitted keyframes (by sequence index): %s", keyframe_ids) - logger.info("Loaded %d admitted keyframes from track pairs", len(keyframe_ids)) + values = read_single_dataset(admitted_frames_path, admission_metadata) + if values.ndim != 1 or values.dtype != np.dtype("uint32"): + raise CacheMetadataMismatch("Selected frame payload must be a uint32 vector") + frame_ids = tuple(map(int, values)) + indices = frames.indices(frame_ids) + if len(indices) < 2 or indices[0] != 0 or indices[-1] != len(frames.frame_ids) - 1: + raise CacheMetadataMismatch("Selected frames must include both timeline endpoints") + logger.info("Loaded %d admitted frames", len(frame_ids)) if keyframe_options.force_gt_keyframes: - gt_indices = keyframe_selector.get_gt_frame_indices(sequence, scene_parser) - missing_gt = set(gt_indices) - set(keyframe_ids) + gt_indices = keyframe_selector.get_gt_frame_indices(frames.streams[0], scene_parser) + missing_gt = set(gt_indices) - set(indices) if missing_gt: raise AssertionError( f"Cached keyframes missing {len(missing_gt)} GT frames (out of {len(gt_indices)} total).\n" f"Missing GT indices: {sorted(missing_gt)}\n" "This should not happen with the fixed code. Re-run with --force-frontend to regenerate keyframes." ) - - salient_metadata = salient_feature_cache_metadata( - salient_options=salient_options, - sequence=sequence, - timestamps=timestamps, - track_pairs_metadata=read_cache_metadata(track_pairs_path), - ) - expected_names = [sequence[index] for index in keyframe_ids] - if incremental_cache_is_complete( - salient_features_path, - salient_metadata, - expected_names, - ): - return names - logger.info("Cached admission and track-propagation section is incomplete") - return None + return frame_ids def commit_keyframes( *, - sequence, - timestamps, + frames, keyframe_ids, gt_frame_indices, - track_pairs_path, - salient_features_path, + admitted_frames_path, keyframe_options, - salient_options, - track_pairs_metadata, + admission_metadata, ): + frame_ids = tuple(frames.frame_ids[index] for index in keyframe_ids) count_message = f"Number of detected keyframes: {len(keyframe_ids)}" if keyframe_options.force_gt_keyframes: count_message += f" (including {sum(index in gt_frame_indices for index in keyframe_ids)} GT frames)" logger.debug("Detected keyframes (by sequence index): %s", keyframe_ids) logger.info(count_message) - expected_names = [sequence[index] for index in keyframe_ids] - track_pairs = tuple(zip(expected_names, expected_names[1:])) - write_pair_artifact(track_pairs_path, track_pairs, track_pairs_metadata) - prune_incremental_items(salient_features_path, expected_names) - final_metadata = salient_feature_cache_metadata( - salient_options=salient_options, - sequence=sequence, - timestamps=timestamps, - track_pairs_metadata=read_cache_metadata(track_pairs_path), - ) - mark_incremental_cache_complete(salient_features_path, final_metadata, expected_names) - logger.info("Cached admitted track pairs to: %s", track_pairs_path) + write_single_dataset(admitted_frames_path, np.asarray(frame_ids, dtype=np.uint32), admission_metadata) + logger.info("Cached admitted frames to: %s", admitted_frames_path) diff --git a/vidmap/frontend/keyframes/processing.py b/vidmap/frontend/keyframes/processing.py index 6049e38..7f9f583 100644 --- a/vidmap/frontend/keyframes/processing.py +++ b/vidmap/frontend/keyframes/processing.py @@ -4,22 +4,29 @@ import logging from dataclasses import dataclass as result_dataclass +from dataclasses import field +from itertools import chain from pathlib import Path from typing import TYPE_CHECKING +import h5py from tqdm import tqdm if TYPE_CHECKING: import torch -from vidmap.datasets.base import DatasetParser +from vidmap.datasets.capture import CaptureSnapshot from vidmap.frontend.cache import ( - CacheMetadataMismatch, CompleteArtifactContract, + attach_incremental_cache_identity, certify_complete_artifact, + inspect_incremental_items, + mark_incremental_cache_complete, prepare_incremental_cache, + prune_incremental_items, + read_cache_metadata, ) -from vidmap.frontend.correspondences import validate_pair_name_plan +from vidmap.frontend.correspondences import combine_and_deduplicate_pairs from vidmap.frontend.geocalib import ( estimate_keyframe_bootstrap_intrinsics, validate_keyframe_bootstrap_frame_dimensions, @@ -39,25 +46,40 @@ logger = logging.getLogger(__name__) -@result_dataclass(frozen=True) +@result_dataclass class KeyframePlan: """Certified keyframe selection and its consecutive tracking plan.""" - names: tuple[str, ...] - track_pairs: tuple[tuple[str, str], ...] - track_pairs_artifact: CompleteArtifactContract + frame_ids: tuple[int, ...] + frames: FrameSequence + admission_artifact: CompleteArtifactContract + streams: tuple[tuple[str, ...], ...] = field(init=False) + same_frame_flags: dict = field(init=False) + names: tuple[str, ...] = field(init=False) + track_pairs: tuple[tuple[str, str], ...] = field(init=False) + frame_camera_indices: dict = field(init=False) def __post_init__(self): - names = tuple(self.names) - track_pairs = tuple(tuple(pair) for pair in self.track_pairs) - if len(names) < 2 or len(set(names)) != len(names): - raise ValueError("Keyframe plan requires at least two unique images") - expected_pairs = tuple(zip(names, names[1:])) - if track_pairs != expected_pairs: - raise ValueError("Keyframe track pairs must be the exact adjacent image chain") - validate_pair_name_plan(track_pairs) - object.__setattr__(self, "names", names) - object.__setattr__(self, "track_pairs", track_pairs) + self.frame_ids = tuple(self.frame_ids) + self.streams = self.frames.select(self.frames.indices(self.frame_ids)) + if len(self.frame_ids) < 2: + raise ValueError("Keyframe selection must contain at least two native frames") + self.names = tuple(chain.from_iterable(self.streams)) + self.track_pairs = tuple(pair for stream in self.streams for pair in zip(stream, stream[1:])) + self.same_frame_flags = { + (frame[first], frame[second]): flag + for frame in zip(*self.streams, strict=True) + for first, second, flag in self.frames.same_frame_stream_pairs + } + self.frame_camera_indices = { + name: (index, camera_id) + for stream, camera_id in zip(self.streams, self.frames.stream_camera_ids, strict=True) + for index, name in enumerate(stream) + } + + @property + def same_frame_pairs(self): + return tuple(combine_and_deduplicate_pairs(tuple(self.same_frame_flags))) @result_dataclass(frozen=True) @@ -76,7 +98,7 @@ class KeyframeProcessor: def __init__( self, *, - scene_parser: DatasetParser, + input_snapshot: CaptureSnapshot, frames: FrameSequence, paths: FrontendPaths, force_recompute: bool, @@ -88,7 +110,8 @@ def __init__( device: torch.device, ): self.device = device - self.scene_parser = scene_parser + self.scene_parser = input_snapshot.capture + self.input_snapshot = input_snapshot self.frames = frames self.paths = paths self.force_recompute = force_recompute @@ -98,39 +121,32 @@ def __init__( self.keyframe_options = keyframe_options self.salient_options = salient_options - def load_cached(self, track_pairs_metadata) -> KeyframePlan | None: - """Load the admitted plan from its adjacent track-pair chain.""" + def load_cached(self, admission_metadata) -> KeyframePlan | None: + """Load the frame selection and project it onto current camera streams.""" if self.keyframe_options.intrinsics_source == "geocalib": validate_keyframe_bootstrap_frame_dimensions( self.scene_parser.rgb_dir, - self.frames.names, + self.frames.streams[0], ) - keyframe_names = keyframe_cache.load_cached_names( + frame_ids = keyframe_cache.load_cached_frame_ids( + frames=self.frames, scene_parser=self.scene_parser, - sequence=self.frames.names, - timestamps=self.frames.timestamps, - track_pairs_path=self.paths.track_pairs_path, - salient_features_path=self.paths.salient_features_path, - force_recompute=self.force_recompute, keyframe_options=self.keyframe_options, - salient_options=self.salient_options, - track_pairs_metadata=track_pairs_metadata, + admitted_frames_path=self.paths.admitted_frames_path, + force_recompute=self.force_recompute, + admission_metadata=admission_metadata, ) - if keyframe_names is None: + if frame_ids is None: return None - plan = self._certified_plan(keyframe_names, track_pairs_metadata) + plan = self._certified_plan(frame_ids, admission_metadata) self._write_repro_plan(plan) return plan - def _certified_plan(self, keyframe_sequence, track_pairs_metadata) -> KeyframePlan: - keyframe_sequence = tuple(keyframe_sequence) - keyframe_pairs = tuple(zip(keyframe_sequence, keyframe_sequence[1:])) - if len(keyframe_sequence) < 2 or not keyframe_pairs: - raise CacheMetadataMismatch("Frontend requires at least two keyframes and one track pair") + def _certified_plan(self, frame_ids, admission_metadata) -> KeyframePlan: return KeyframePlan( - names=keyframe_sequence, - track_pairs=keyframe_pairs, - track_pairs_artifact=certify_complete_artifact(self.paths.track_pairs_path, track_pairs_metadata), + frame_ids=frame_ids, + frames=self.frames, + admission_artifact=certify_complete_artifact(self.paths.admitted_frames_path, admission_metadata), ) def _write_repro_plan(self, plan: KeyframePlan) -> None: @@ -149,7 +165,7 @@ def _write_repro_plan(self, plan: KeyframePlan) -> None: def select_candidates(self, salient_metadata) -> KeyframeCandidates: """Select low-resolution candidates without publishing final keyframes.""" - sequence = list(self.frames.names) + sequence = list(self.frames.streams[0]) keyframe_options = self.keyframe_options scene_parser = self.scene_parser bootstrap_intrinsics = None @@ -164,9 +180,15 @@ def select_candidates(self, salient_metadata) -> KeyframeCandidates: prepare_incremental_cache( self.paths.salient_features_path, salient_metadata, - overwrite=True, + overwrite=self.force_recompute, ) + attach_incremental_cache_identity(self.paths.salient_features_path, salient_metadata) + with h5py.File(self.paths.salient_features_path, "a") as hfile: + for name in sequence: + if name in hfile: + del hfile[name] + loader, total_pairs, original_width, original_height = keyframe_matching.build_pair_loader( scene_parser, sequence, @@ -211,6 +233,10 @@ def select_candidates(self, salient_metadata) -> KeyframeCandidates: progress.update(1) keyframe_ids = selector.finish() + with h5py.File(self.paths.salient_features_path, "a") as hfile: + for name in sequence: + if name in hfile: + hfile[name].attrs["input_fingerprint"] = self.input_snapshot.image_sha256[name] if keyframe_options.intrinsics_source == "geocalib": calibrations = {index: bootstrap_intrinsics for index in keyframe_ids} else: @@ -218,7 +244,7 @@ def select_candidates(self, salient_metadata) -> KeyframeCandidates: calibrations = {index: calibration_plan[index][1] for index in keyframe_ids} return KeyframeCandidates( indices=tuple(keyframe_ids), - names=tuple(self.frames.names[index] for index in keyframe_ids), + names=tuple(sequence[index] for index in keyframe_ids), forced_indices=frozenset(selector.gt_frame_indices), calibrations=calibrations, ) @@ -227,20 +253,65 @@ def select_candidates(self, salient_metadata) -> KeyframeCandidates: del aliked_model empty_device_cache() - def commit_keyframes(self, admitted_ids, forced_ids, track_pairs_metadata) -> KeyframePlan: + def commit_keyframes(self, admitted_ids, forced_ids, admission_metadata) -> KeyframePlan: """Publish the candidates retained by lookahead admission.""" keyframe_cache.commit_keyframes( - sequence=self.frames.names, - timestamps=self.frames.timestamps, + frames=self.frames, keyframe_ids=admitted_ids, gt_frame_indices=forced_ids, - track_pairs_path=self.paths.track_pairs_path, - salient_features_path=self.paths.salient_features_path, + admitted_frames_path=self.paths.admitted_frames_path, keyframe_options=self.keyframe_options, - salient_options=self.salient_options, - track_pairs_metadata=track_pairs_metadata, + admission_metadata=admission_metadata, ) - names = tuple(self.frames.names[index] for index in admitted_ids) - plan = self._certified_plan(names, track_pairs_metadata) + frame_ids = tuple(self.frames.frame_ids[index] for index in admitted_ids) + plan = self._certified_plan(frame_ids, admission_metadata) self._write_repro_plan(plan) return plan + + def complete_salient_features(self, plan): + """Complete only missing or changed selected-image features.""" + metadata = keyframe_cache.salient_feature_cache_metadata(salient_options=self.salient_options) + path = self.paths.salient_features_path + prepare_incremental_cache(path, metadata) + cached = read_cache_metadata(path) + complete = "complete" in cached and cached["complete"] + sources = {name: self.input_snapshot.image_sha256[name] for name in plan.names} + with h5py.File(path, "r") as hfile: + stale = [ + name + for name in plan.names + if name in hfile + and ( + "input_fingerprint" not in hfile[name].attrs + or hfile[name].attrs["input_fingerprint"] != sources[name] + ) + ] + if not stale and complete and cached["expected_items"] == list(plan.names): + return + attach_incremental_cache_identity(path, metadata) + with h5py.File(path, "a") as hfile: + for name in stale: + del hfile[name] + prune_incremental_items(path, plan.names) + _, missing = inspect_incremental_items(path, plan.names, metadata, repair_malformed=True) + self._extract_other_views(missing) + with h5py.File(path, "a") as hfile: + for name in missing: + hfile[name].attrs["input_fingerprint"] = sources[name] + mark_incremental_cache_complete(path, metadata, plan.names) + + def _extract_other_views(self, names): + """Extract side-camera salient features only after whole-frame admission.""" + from vidmap.frontend.keyframes.selection import compute_aliked_features_for_frame + + if not names: + return + model = keyframe_selector.create_aliked_model(self.salient_options, device=self.device) + try: + for name in names: + compute_aliked_features_for_frame( + self.scene_parser, self.paths.salient_features_path, name, model, self.salient_options + ) + finally: + del model + empty_device_cache(self.device) diff --git a/vidmap/frontend/keyframes/selector.py b/vidmap/frontend/keyframes/selector.py index 4fc8973..a37e35c 100644 --- a/vidmap/frontend/keyframes/selector.py +++ b/vidmap/frontend/keyframes/selector.py @@ -14,12 +14,8 @@ def get_gt_frame_indices(sequence, scene_parser): """Return sequence indices for frames that have ground truth poses.""" - gt_names = set() - total_images = 0 - for image in scene_parser.rec.images.values(): - total_images += 1 - if image.has_pose: - gt_names.add(image.name) + gt_names = scene_parser.forced_keyframe_names + total_images = len(scene_parser.rec.images) gt_indices = [] for idx, name in enumerate(sequence): diff --git a/vidmap/frontend/loop_closure/cache.py b/vidmap/frontend/loop_closure/cache.py index f1daa13..4dc1d2c 100644 --- a/vidmap/frontend/loop_closure/cache.py +++ b/vidmap/frontend/loop_closure/cache.py @@ -19,8 +19,8 @@ inspect_incremental_items, mark_incremental_cache_complete, prepare_incremental_cache, - prune_incremental_items, ) +from vidmap.frontend.cache.validation import dataset_parent_names from vidmap.frontend.correspondences import canonical_pair_names from vidmap.frontend.loop_closure.matching import collect_cached_extended_matches, repair_extended_match_pairs @@ -28,27 +28,29 @@ class ExtendedMatchCache: """Create, publish, repair, consume, and certify extended matches.""" - def __init__(self, paths, metadata, *, force_recompute): + def __init__(self, paths, *, force_recompute): self.paths = paths self.path = paths.extended_matches_path - self.metadata = metadata self.force_recompute = force_recompute - def prepare(self): + def prepare(self, metadata): + self.metadata = metadata prepare_incremental_cache( self.path, self.metadata, overwrite=self.force_recompute, ) - def publish_staging(self, staging_path): - """Atomically publish staged sequential matches while retaining valid cached pairs.""" - if not self.force_recompute and cache_is_valid(self.path, self.metadata): + def publish_staging(self, staging_path, metadata): + """Bind private samples to completed sparse indices, retaining compatible pairs.""" + self.metadata = metadata + if not self.force_recompute and cache_is_valid(self.path, metadata): with h5py.File(self.path, "r") as source, h5py.File(staging_path, "a") as destination: - for name in source: - if name not in destination: - source.copy(name, destination) - attach_incremental_cache_identity(staging_path, self.metadata) + for pair in dataset_parent_names(source): + if pair not in destination: + parent, name = pair.rsplit("/", 1) + source.copy(pair, destination.require_group(parent), name=name) + attach_incremental_cache_identity(staging_path, metadata) os.replace(staging_path, self.path) def repair( @@ -84,15 +86,20 @@ def collect(self, pairs, *, match_threshold): ) def complete(self, pairs) -> IncrementalArtifactContract: - expected_items = tuple(canonical_pair_names(pairs)) - _, missing_items = inspect_incremental_items( + requested_items = set(canonical_pair_names(pairs)) + with h5py.File(self.path, "r") as hfile: + stored_items = set(dataset_parent_names(hfile)) + present, missing = inspect_incremental_items( self.path, - expected_items, + sorted(stored_items | requested_items), self.metadata, repair_malformed=True, ) - if missing_items: - raise CacheMetadataMismatch(f"{self.path}: missing {len(missing_items)} planned extended-match pairs") - prune_incremental_items(self.path, expected_items) + missing_requested = requested_items.intersection(missing) + if missing_requested: + raise CacheMetadataMismatch(f"{self.path}: missing {len(missing_requested)} planned extended-match pairs") + # Selection owns the requested plan; inference retains valid unselected + # pairs so a later plan expansion does not repeat matching. + expected_items = tuple(present) mark_incremental_cache_complete(self.path, self.metadata, expected_items) return certify_incremental_artifact(self.path, self.metadata, expected_items) diff --git a/vidmap/frontend/loop_closure/extended_matches.py b/vidmap/frontend/loop_closure/extended_matches.py index d6db5ca..23faa1e 100644 --- a/vidmap/frontend/loop_closure/extended_matches.py +++ b/vidmap/frontend/loop_closure/extended_matches.py @@ -13,7 +13,7 @@ import numpy as np -from vidmap.datasets.base import DatasetParser +from vidmap.datasets.capture import PreparedCapture from vidmap.frontend.cache import ( CompleteArtifactContract, IncrementalArtifactContract, @@ -33,6 +33,7 @@ validate_correspondence_alignment, validate_pair_name_plan, ) +from vidmap.frontend.keyframes.processing import KeyframePlan from vidmap.frontend.loop_closure.matching import merge_matches from vidmap.frontend.loop_closure.retrieval import ( RETRIEVAL_PAIR_SELECTION_POLICY_VERSION, @@ -70,7 +71,14 @@ def __post_init__(self): def retrieval_pairs_cache_metadata( - extended_options, sequence, sequential_pairs, tcorr, retrieval_features_fingerprint, *, tcorr_min_matches + extended_options, + sequence, + sequential_pairs, + tcorr, + retrieval_features_fingerprint, + *, + tcorr_min_matches, + frame_camera_indices, ): conf = extended_options return cache_metadata( @@ -83,11 +91,14 @@ def retrieval_pairs_cache_metadata( "tcorr_min_matches": tcorr_min_matches, "lc_pair_nms": conf.lc_pair_nms, "lc_pair_nms_radius": conf.lc_pair_nms_radius, + "rig_policy": "frame-neighborhood-camera-nms-v1", + "max_pairs_per_frame_pair": conf.max_pairs_per_frame_pair, }, ordered_inputs={ "sequence": sequence, "sequential_pairs": sequential_pairs, "tcorr": tcorr, + "frame_camera_indices": frame_camera_indices, }, upstream={"retrieval_features": retrieval_features_fingerprint}, payload_format="ordered-image-pairs", @@ -101,12 +112,13 @@ class ExtendedMatchBuilder: def __init__( self, *, - scene_parser: DatasetParser, + scene_parser: PreparedCapture, paths: FrontendPaths, force_recompute: bool, repro_dir: Path | None, tracker: LazyRoMaV2Tracker, tracks: SparseTrackResult, + keyframes: KeyframePlan, transitive: TransitiveCorrespondenceResult, highres_options: RoMaImageOptions, lowres_match_resolution: int, @@ -122,6 +134,7 @@ def __init__( self.repro_dir = repro_dir self.tracker = tracker self.tracks = tracks + self.keyframes = keyframes self.transitive = transitive self.highres_options = highres_options self.lowres_match_resolution = lowres_match_resolution @@ -132,9 +145,10 @@ def __init__( def _build_retrieval_pairs(self): """Compute retrieval features and loop-closure matches.""" paths = self.paths - sequence = self.transitive.sequence + sequence = self.keyframes.names sequential_pairs = self.tracks.sequential_pairs - tcorr = self.transitive.tcorr + timeline = self.keyframes.frame_camera_indices + tcorr = {pair: len(rows) for pair, rows in self.transitive.tcorr.items()} from vidmap.utils.profiling import log_memory, record_timing, sync_time retrieval_pairs = [] @@ -162,6 +176,7 @@ def _build_retrieval_pairs(self): tcorr, retrieval_features_fingerprint, tcorr_min_matches=tcorr_min_matches, + frame_camera_indices=timeline, ) if ( @@ -192,10 +207,11 @@ def _build_retrieval_pairs(self): tcorr, retrieval_features_fingerprint, tcorr_min_matches=tcorr_min_matches, + frame_camera_indices=timeline, ) retrieval_pairs = generate_retrieval_pairs( sequence=sequence, - tcorr=tcorr, + tcorr_counts=tcorr, sequential_pairs=sequential_pairs, retrieval_path=paths.retrieval_features_path, tcorr_min_matches=tcorr_min_matches, @@ -204,11 +220,14 @@ def _build_retrieval_pairs(self): lc_pair_nms=ext_conf.lc_pair_nms, lc_pair_nms_radius=ext_conf.lc_pair_nms_radius, device=self.device, + frame_camera_indices=timeline, + max_pairs_per_frame_pair=ext_conf.max_pairs_per_frame_pair, ) - validate_pair_name_plan((*sequential_pairs, *retrieval_pairs)) write_pair_artifact(paths.retrieval_pairs_path, retrieval_pairs, retrieval_pairs_metadata) logger.info("Cached %d retrieval pairs", len(retrieval_pairs)) validate_pair_name_plan((*sequential_pairs, *retrieval_pairs)) + if any(timeline[first][0] == timeline[second][0] for first, second in retrieval_pairs): + raise ValueError("Rig retrieval pairs must not connect images in the same frame") if self.repro_dir is not None: write_pair_order_artifact( self.repro_dir / "stage1_retrieval_pair_order.json", @@ -216,28 +235,12 @@ def _build_retrieval_pairs(self): label="retrieval_pairs", ) - started = sync_time() - _present_count, missing_count = self.tracks.extended_matches.repair( - retrieval_pairs, - label="retrieval", - tracker=self.tracker, - scene_parser=self.scene_parser, - image_options=self.highres_options, - lowres_match_resolution=self.lowres_match_resolution, - match_threshold=ext_conf.lc_match_thresh, - device=self.device, - ) - if missing_count: - record_timing("lc_streaming", sync_time() - started) - log_memory("lc_streaming") - retrieval_pairs_artifact = certify_complete_artifact(paths.retrieval_pairs_path, retrieval_pairs_metadata) return retrieval_pairs, retrieval_pairs_artifact - def _finalize_extended_matches(self, retrieval_pairs): + def _finalize_extended_matches(self, extended_pairs): """Merge persisted extended matches and certify their complete pair plan.""" tcorr = dict(self.transitive.tcorr) - extended_pairs = combine_and_deduplicate_pairs(self.tracks.sequential_pairs, retrieval_pairs) if self.repro_dir is not None: write_pair_order_artifact( self.repro_dir / "stage1_extended_pairs.json", @@ -256,6 +259,10 @@ def _finalize_extended_matches(self, retrieval_pairs): merged_tcorr = tcorr lc_masks = {pair: np.zeros(len(matches), dtype=bool) for pair, matches in tcorr.items()} + explicit = {frozenset(pair): flag for pair, flag in self.keyframes.same_frame_flags.items()} + for pair, mask in lc_masks.items(): + if frozenset(pair) in explicit: + mask.fill(not explicit[frozenset(pair)]) if self.repro_dir is not None: write_tcorr_artifact( self.repro_dir / "stage1_merged_tcorr_lc_masks.json", @@ -268,12 +275,34 @@ def _finalize_extended_matches(self, retrieval_pairs): def build(self) -> ExtendedMatchResult: retrieval_pairs, retrieval_pairs_artifact = self._build_retrieval_pairs() - tcorr, lc_masks = self._finalize_extended_matches(retrieval_pairs) - extended_pairs = combine_and_deduplicate_pairs(self.tracks.sequential_pairs, retrieval_pairs) + # First occurrence owns inference direction: temporal flow first, then + # explicit same-frame requests, then retrieval. Storage stays canonical. + requests = {} + for pair in (*self.tracks.sequential_pairs, *self.keyframes.same_frame_pairs, *retrieval_pairs): + requests.setdefault(frozenset(pair), pair) + extended_pairs = combine_and_deduplicate_pairs(requests.values()) + from vidmap.utils.profiling import log_memory, record_timing, sync_time + + started = sync_time() + _present_count, missing_count = self.tracks.extended_matches.repair( + tuple(requests.values()), + label="extended", + tracker=self.tracker, + scene_parser=self.scene_parser, + image_options=self.highres_options, + lowres_match_resolution=self.lowres_match_resolution, + match_threshold=self.extended_options.lc_match_thresh, + device=self.device, + ) + if missing_count: + record_timing("lc_streaming", sync_time() - started) + log_memory("lc_streaming") + artifact = self.tracks.extended_matches.complete(extended_pairs) + tcorr, lc_masks = self._finalize_extended_matches(extended_pairs) return ExtendedMatchResult( retrieval_pairs=tuple(retrieval_pairs), retrieval_pairs_artifact=retrieval_pairs_artifact, tcorr=tcorr, lc_masks=lc_masks, - artifact=self.tracks.extended_matches.complete(extended_pairs), + artifact=artifact, ) diff --git a/vidmap/frontend/loop_closure/matching.py b/vidmap/frontend/loop_closure/matching.py index eca6247..178fd91 100644 --- a/vidmap/frontend/loop_closure/matching.py +++ b/vidmap/frontend/loop_closure/matching.py @@ -15,11 +15,12 @@ import h5py import numpy as np import torch +from natsort import natsorted from torch.utils.data import DataLoader, Dataset from tqdm import tqdm from vidmap.frontend.cache import CacheMetadataMismatch, inspect_incremental_items -from vidmap.frontend.correspondences import canonical_pair_names +from vidmap.frontend.correspondences import canonical_pair_names, combine_and_deduplicate_pairs from vidmap.frontend.h5_write_queue import H5WriteQueue, write_pair_matches from vidmap.frontend.image_dataset import ImageDatasetOptions, get_image_size from vidmap.frontend.tracking.kernels import select_lc_matches_from_dense @@ -28,6 +29,8 @@ from vidmap.utils.logging import progress_bars_enabled from vidmap.utils.parsers import names_to_pair +EXTENDED_MATCH_DIRECTION_POLICY = "temporal-tracking-selected-retrieval-canonical-storage-v2" + logger = logging.getLogger(__name__) @@ -89,6 +92,43 @@ def collate_retrieval_pair(batch): return batch[0] +def queue_extended_matches( + name0, name1, keypoints0, keypoints1, matches_dense, certainty, *, scene_parser, match_threshold, writer_queue +): + """Sample in inference direction, then canonicalize only the persisted rows.""" + lc_result = select_lc_matches_from_dense( + keypoints0, + keypoints1, + matches_dense, + certainty, + source_size=get_image_size(scene_parser, name0), + target_size=get_image_size(scene_parser, name1), + lc_match_thresh=match_threshold, + ) + matches0 = lc_result["matches0"] + matching_scores0 = lc_result["matching_scores0"] + pair = tuple(natsorted((name0, name1))) + if pair != (name0, name1): + source = np.flatnonzero(matches0 >= 0) + target = matches0[source] + matches = np.full(len(keypoints1), -1, dtype=np.int32) + scores = np.zeros(len(keypoints1), dtype=np.float32) + matches[target] = source + scores[target] = matching_scores0[source] + matches0 = matches + matching_scores0 = scores + writer_queue.put( + ( + names_to_pair(*pair), + { + "matches0": torch.from_numpy(matches0)[None], + "matching_scores0": torch.from_numpy(matching_scores0).float()[None], + }, + ) + ) + return int((matches0 >= 0).sum()) + + def _match_loop_closures_streaming( scene_parser, sparse_features_path, @@ -98,8 +138,8 @@ def _match_loop_closures_streaming( lowres_match_resolution, *, device: torch.device, + lc_writer_queue, lc_match_thresh=0.5, - lc_writer_queue=None, ): """ Streaming LC frontend for retrieval pairs. @@ -116,7 +156,7 @@ def _match_loop_closures_streaming( lowres_match_resolution: Processing resolution for lowres matches device: Execution device lc_match_thresh: Certainty threshold for LC matches - lc_writer_queue: Optional H5WriteQueue instance + lc_writer_queue: H5WriteQueue instance """ if not retrieval_pairs: return @@ -221,37 +261,17 @@ def _match_loop_closures_streaming( kpts0 = keypoint_reader.get(name0) kpts1 = keypoint_reader.get(name1) - # Get dimensions - W0, H0 = get_image_size(scene_parser, name0) - W1, H1 = get_image_size(scene_parser, name1) - - # === Core LC frontend === - lc_result = select_lc_matches_from_dense( + _stats["total_matches"] += queue_extended_matches( + name0, + name1, kpts0, kpts1, matches_dense, certainty, - source_size=(W0, H0), - target_size=(W1, H1), - lc_match_thresh=lc_match_thresh, + scene_parser=scene_parser, + match_threshold=lc_match_thresh, + writer_queue=lc_writer_queue, ) - matches0 = lc_result["matches0"] - matching_scores0 = lc_result["matching_scores0"] - - num_matches = (matches0 >= 0).sum() - _stats["total_matches"] += num_matches - - # Also write to disk if caching is enabled - if lc_writer_queue is not None: - lc_writer_queue.put( - ( - pair_name, - { - "matches0": torch.from_numpy(matches0)[None], - "matching_scores0": torch.from_numpy(matching_scores0).float()[None], - }, - ) - ) # Free tensors from this iteration del matches_dense, certainty @@ -304,7 +324,7 @@ def repair_extended_match_pairs( device: torch.device, ): """Repair and verify one planned set of extended-match pairs.""" - expected_names = canonical_pair_names(pairs) + expected_names = canonical_pair_names(combine_and_deduplicate_pairs(pairs)) present_names, missing_names = inspect_incremental_items( paths.extended_matches_path, expected_names, @@ -312,7 +332,7 @@ def repair_extended_match_pairs( repair_malformed=True, ) missing_name_set = set(missing_names) - missing_pairs = [pair for pair in pairs if names_to_pair(*pair) in missing_name_set] + missing_pairs = [pair for pair in pairs if names_to_pair(*natsorted(pair)) in missing_name_set] if missing_pairs: logger.info("Streaming LC frontend for %d %s pairs", len(missing_pairs), label) match_loop_closures_streaming( diff --git a/vidmap/frontend/loop_closure/retrieval.py b/vidmap/frontend/loop_closure/retrieval.py index 31ccf57..3c56201 100644 --- a/vidmap/frontend/loop_closure/retrieval.py +++ b/vidmap/frontend/loop_closure/retrieval.py @@ -3,6 +3,7 @@ import logging import pprint from collections import defaultdict +from itertools import chain from pathlib import Path import h5py @@ -22,7 +23,7 @@ from vidmap.utils.device import empty_device_cache from vidmap.utils.logging import progress_bars_enabled -RETRIEVAL_PAIR_SELECTION_POLICY_VERSION = 1 +RETRIEVAL_PAIR_SELECTION_POLICY_VERSION = 3 logger = logging.getLogger(__name__) @@ -164,33 +165,24 @@ def _pairs_from_score_matrix(scores, invalid, num_select, min_score, return_scor return pairs -def _retrieve_pairs(path, reference_query_dict, num_matched, min_score, return_scores, *, device: torch.device): +def _retrieve_pairs( + path, names, reference_query_pairs, num_matched, min_score, return_scores, *, device: torch.device +): with h5py.File(str(path), "r", libver="latest") as hfile: - - def has_descriptor(name): - return name in hfile and "global_descriptor" in hfile[name] - - references = [name for name in reference_query_dict if has_descriptor(name)] - queries = list( - dict.fromkeys( - query for candidates in reference_query_dict.values() for query in candidates if has_descriptor(query) - ) - ) - if not references or not queries: + available = [name for name in dict.fromkeys(names) if name in hfile and "global_descriptor" in hfile[name]] + if not available: return [] - reference_descriptors = _load_descriptors(references, hfile).to(device) - query_descriptors = _load_descriptors(queries, hfile).to(device) - reference_indices = {name: index for index, name in enumerate(references)} - query_indices = {name: index for index, name in enumerate(queries)} + descriptors = _load_descriptors(available, hfile).to(device) + indices = {name: index for index, name in enumerate(available)} pairs = [] - for reference, candidates in reference_query_dict.items(): - if reference not in reference_indices: + for reference, candidates in reference_query_pairs: + if reference not in indices: continue - valid_queries = [query for query in candidates if query in query_indices] + valid_queries = [query for query in candidates if query in indices] if not valid_queries: continue - reference_descriptor = reference_descriptors[reference_indices[reference] : reference_indices[reference] + 1] - candidate_descriptors = query_descriptors[[query_indices[query] for query in valid_queries]] + reference_descriptor = descriptors[indices[reference] : indices[reference] + 1] + candidate_descriptors = descriptors[[indices[query] for query in valid_queries]] similarities = torch.einsum("id,jd->ij", reference_descriptor, candidate_descriptors) self_mask = np.array([reference])[:, None] == np.array(valid_queries)[None] pair_indices = _pairs_from_score_matrix( @@ -212,7 +204,7 @@ def has_descriptor(name): def generate_retrieval_pairs( sequence, - tcorr, + tcorr_counts, sequential_pairs, retrieval_path, tcorr_min_matches, @@ -220,39 +212,53 @@ def generate_retrieval_pairs( nquery, lc_pair_nms=False, lc_pair_nms_radius=2, + max_pairs_per_frame_pair: int | None = None, *, + frame_camera_indices, device: torch.device, ): """Generate retrieval pairs excluding sequential and sufficiently tracked pairs.""" if not retrieval_path.exists(): raise ValueError("Retrieval features not found. Run compute_retrieval_features step first.") - sequential_pairs_set = {frozenset(pair) for pair in sequential_pairs} - tcorr_pairs_set = {frozenset(pair) for pair, matches in tcorr.items() if len(matches) > tcorr_min_matches} - untracked_pairs = defaultdict(list) - for id_a in range(len(sequence)): - for id_b in range(len(sequence)): - if id_a == id_b: - continue - pair = frozenset([sequence[id_a], sequence[id_b]]) - if pair not in tcorr_pairs_set and pair not in sequential_pairs_set: - untracked_pairs[sequence[id_a]].append(sequence[id_b]) - - if not untracked_pairs: + excluded_images = defaultdict(set) + for first, second in chain( + sequential_pairs, (pair for pair, count in tcorr_counts.items() if count > tcorr_min_matches) + ): + excluded_images[first].add(second) + excluded_images[second].add(first) + + def candidates(): + for reference in sequence: + excluded = excluded_images[reference] + frame = frame_camera_indices[reference][0] + queries = [name for name in sequence if name not in excluded and frame_camera_indices[name][0] != frame] + if queries: + yield reference, queries + + reference_queries = iter(candidates()) + first = next(reference_queries, None) + if first is None: logger.info("No untracked pairs found for retrieval") return [] + limit_frame_pairs = max_pairs_per_frame_pair is not None retrieval_pairs = _retrieve_pairs( retrieval_path, - untracked_pairs, + sequence, + chain((first,), reference_queries), nquery, retrieval_min_score, - lc_pair_nms, + lc_pair_nms or limit_frame_pairs, device=device, ) + if limit_frame_pairs: + retrieval_pairs = _limit_rig_frame_pairs(retrieval_pairs, frame_camera_indices, max_pairs_per_frame_pair) if lc_pair_nms: raw_count = len(retrieval_pairs) - retrieval_pairs = endpoint_nms_retrieval_pairs(retrieval_pairs, sequence, radius=lc_pair_nms_radius) + retrieval_pairs = endpoint_nms_retrieval_pairs( + retrieval_pairs, sequence, radius=lc_pair_nms_radius, frame_camera_indices=frame_camera_indices + ) logger.info( "LC pair endpoint NMS radius=%d: %d -> %d pairs", lc_pair_nms_radius, @@ -260,18 +266,20 @@ def generate_retrieval_pairs( len(retrieval_pairs), ) else: + if limit_frame_pairs: + retrieval_pairs = [(name0, name1) for name0, name1, _ in retrieval_pairs] retrieval_pairs = natsorted(tuple(natsorted(pair)) for pair in {frozenset(pair) for pair in retrieval_pairs}) logger.info("Generated %d retrieval pairs, excluding sequential pairs", len(retrieval_pairs)) return retrieval_pairs -def endpoint_nms_retrieval_pairs(scored_pairs, sequence, radius): +def endpoint_nms_retrieval_pairs(scored_pairs, sequence, radius, frame_camera_indices): """Keep the best retrieval edge per local query/target neighborhood.""" if radius < 0: raise ValueError(f"lc_pair_nms_radius must be non-negative, got {radius}") - seq_index = {name: idx for idx, name in enumerate(sequence)} + seq_index = {name: frame_camera_indices[name][0] for name in sequence} best_by_pair = {} for name0, name1, score in scored_pairs: if name0 not in seq_index or name1 not in seq_index or name0 == name1: @@ -292,14 +300,31 @@ def endpoint_nms_retrieval_pairs(scored_pairs, sequence, radius): key=lambda item: (-item[0], item[2][0], item[2][1]), ) accepted = [] - accepted_endpoints = [] + accepted_endpoints = defaultdict(list) for _score, endpoints, pair in candidates: + camera_pair = tuple(frame_camera_indices[name][1] for name in pair) suppress = any( abs(endpoints[0] - previous[0]) <= radius and abs(endpoints[1] - previous[1]) <= radius - for previous in accepted_endpoints + for previous in accepted_endpoints[camera_pair] ) if suppress: continue accepted.append(pair) - accepted_endpoints.append(endpoints) + accepted_endpoints[camera_pair].append(endpoints) + # Preserve temporal source/target order for inference; only persisted match + # rows are canonicalized, after sampling the directed dense field. return natsorted(accepted) + + +def _limit_rig_frame_pairs(scored_pairs, frame_camera_indices, maximum): + """Bound verification per exposure pair while keeping the strongest views.""" + accepted, seen, counts = [], set(), defaultdict(int) + for name0, name1, score in sorted(scored_pairs, key=lambda item: (-item[2], item[0], item[1])): + image_pair = frozenset((name0, name1)) + frame_pair = frozenset((frame_camera_indices[name0][0], frame_camera_indices[name1][0])) + if image_pair in seen or counts[frame_pair] >= maximum: + continue + seen.add(image_pair) + counts[frame_pair] += 1 + accepted.append((name0, name1, score)) + return accepted diff --git a/vidmap/frontend/options/__init__.py b/vidmap/frontend/options/__init__.py index 5591a48..c34b9b8 100644 --- a/vidmap/frontend/options/__init__.py +++ b/vidmap/frontend/options/__init__.py @@ -3,11 +3,18 @@ from .depth import Da3VideoOptions, DepthEstimationOptions from .keyframes import DetectKeyframesOptions, KeyframeOptions, SalientFeatureOptions from .matching import ExtendedMatchOptions, LowresMatchOptions, PreprocessingOptions, RoMaImageOptions, RoMaV2Options -from .preparation import CameraPriorEstimationOptions, GeomVerifOptions, LCOptions, PreparationOptions +from .preparation import ( + CameraPriorEstimationOptions, + CameraProcessingOptions, + GeomVerifOptions, + LCOptions, + PreparationOptions, +) from .tracking import FrontendTrackOptions, SparseTrackOptions __all__ = [ "CameraPriorEstimationOptions", + "CameraProcessingOptions", "Da3VideoOptions", "DepthEstimationOptions", "DetectKeyframesOptions", diff --git a/vidmap/frontend/options/matching.py b/vidmap/frontend/options/matching.py index 9cfda40..c56ce97 100644 --- a/vidmap/frontend/options/matching.py +++ b/vidmap/frontend/options/matching.py @@ -1,6 +1,7 @@ """Schemas shared by frontend image matching stages.""" -from typing import Optional, Tuple +from dataclasses import field as dc_field +from typing import Annotated, Optional, Tuple from pydantic import ConfigDict, Field, model_validator @@ -67,6 +68,9 @@ class ExtendedMatchOptions: lc_pair_nms: bool = True lc_pair_nms_radius: int = 2 lc_match_thresh: float = 0.05 + max_pairs_per_frame_pair: Annotated[int, Field(ge=1, strict=True)] | None = dc_field( + default=None, metadata={"omit_default": True} + ) def effective_tcorr_min_matches(self, point_budget: int) -> int: return self.tcorr_min_matches * point_budget // CANONICAL_POINT_BUDGET diff --git a/vidmap/frontend/options/preparation.py b/vidmap/frontend/options/preparation.py index 3a47010..d307d17 100644 --- a/vidmap/frontend/options/preparation.py +++ b/vidmap/frontend/options/preparation.py @@ -22,9 +22,19 @@ def validate_applicable_options(self): raise ValueError("Selected-batch inference requires GeoCalib") if self.estimator == "none" and self.initialization != "supplied": raise ValueError("Predicted initialization requires an estimator") + if self.max_images < 1: + raise ValueError("Camera calibration requires at least one sample") return self +@dataclass(frozen=True, config=ConfigDict(extra="forbid")) +class CameraProcessingOptions: + """Complete camera-local settings; share prior groups through config composition.""" + + camera_priors: CameraPriorEstimationOptions + image_mode: Literal["auto", "native", "undistort"] = "auto" + + @dataclass(frozen=True, config=ConfigDict(extra="forbid", strict=True)) class GeomVerifOptions: max_H_inlier_ratio: float = 0.8 diff --git a/vidmap/frontend/options/rig.py b/vidmap/frontend/options/rig.py new file mode 100644 index 0000000..7a782bf --- /dev/null +++ b/vidmap/frontend/options/rig.py @@ -0,0 +1,63 @@ +"""Numeric camera matching policy for synchronized calibrated rigs.""" + +from typing import Annotated + +from pydantic import ConfigDict, Field, model_validator + +from vidmap.configuration.validators import dataclass as options_dataclass + +CameraId = Annotated[int, Field(strict=True, ge=0, lt=2**32 - 1)] +CameraPair = tuple[CameraId, CameraId] +FiniteFloat = Annotated[float, Field(strict=True, allow_inf_nan=False)] + + +@options_dataclass(frozen=True, config=ConfigDict(extra="forbid")) +class RigCameraOptions: + camera_model_name: str + camera_params: tuple[FiniteFloat, ...] + cam_from_rig_rotation: tuple[FiniteFloat, FiniteFloat, FiniteFloat, FiniteFloat] | None = None + cam_from_rig_translation: tuple[FiniteFloat, FiniteFloat, FiniteFloat] | None = None + + @model_validator(mode="after") + def _validate_pose(self): + if self.cam_from_rig_rotation is not None: + if abs(sum(value * value for value in self.cam_from_rig_rotation) - 1) > 1e-6: + raise ValueError("cam_from_rig_rotation must be a unit wxyz quaternion") + return self + + +@options_dataclass(frozen=True, config=ConfigDict(extra="forbid")) +class RigOptions: + """Input calibration and matching policy for numeric camera IDs.""" + + cameras: tuple[RigCameraOptions, ...] = () + frames: str | None = None + + keyframe_camera: CameraId | None = None + match_same_timestamp: tuple[CameraPair, ...] = () + flag_same_timestamp_as_sequential: tuple[CameraPair, ...] = () + + @model_validator(mode="after") + def _validate(self): + for camera_id, camera in enumerate(self.cameras): + if camera_id == 0: + if camera.cam_from_rig_rotation is not None or camera.cam_from_rig_translation is not None: + raise ValueError("Camera 0 is the reference and must not specify cam_from_rig") + elif camera.cam_from_rig_rotation is None or camera.cam_from_rig_translation is None: + raise ValueError("Every non-reference camera requires cam_from_rig rotation and translation") + matching = set() + sequential = set() + for pairs, destination in ( + (self.match_same_timestamp, matching), + (self.flag_same_timestamp_as_sequential, sequential), + ): + for first, second in pairs: + key = frozenset((first, second)) + if len(key) != 2: + raise ValueError("Camera pairs must contain two distinct camera IDs") + if key in destination: + raise ValueError("Duplicate or reversed duplicate camera pair") + destination.add(key) + if not sequential <= matching: + raise ValueError("Sequential same-timestamp pairs must be a subset of matching pairs") + return self diff --git a/vidmap/frontend/paths.py b/vidmap/frontend/paths.py index d672bb4..9b9f879 100644 --- a/vidmap/frontend/paths.py +++ b/vidmap/frontend/paths.py @@ -8,7 +8,7 @@ class FrontendPaths: """Filesystem artifacts produced or consumed by one frontend run.""" - track_pairs_path: Path + admitted_frames_path: Path retrieval_pairs_path: Path sparse_features_path: Path sparse_matches_path: Path @@ -21,6 +21,10 @@ class FrontendPaths: geocalib_per_image_path: Path | None = None geocalib_batch_path: Path | None = None + @property + def lc_masks_path(self) -> Path: + return self.sparse_matches_path.with_name(f"lc_masks-{self.sparse_matches_path.stem}.json") + def _path_component(value, label): value = str(value) @@ -49,7 +53,7 @@ def build_frontend_paths( if cache_variant: trajectory_frontend_dir /= _path_component(cache_variant, "cache_variant") - track_pairs_path = trajectory_frontend_dir / "track_pairs.h5" + admitted_frames_path = trajectory_frontend_dir / "selected_frames.h5" retrieval_pairs_path = trajectory_frontend_dir / "retrieval_pairs.h5" salient_features_path = trajectory_frontend_dir / "salient_features.h5" sparse_features_path = trajectory_frontend_dir / "sparse_features.h5" @@ -63,7 +67,7 @@ def build_frontend_paths( geocalib_batch_path = trajectory_frontend_dir / "geocalib_batch.h5" return FrontendPaths( - track_pairs_path=track_pairs_path, + admitted_frames_path=admitted_frames_path, retrieval_pairs_path=retrieval_pairs_path, sparse_features_path=sparse_features_path, sparse_matches_path=sparse_matches_path, diff --git a/vidmap/frontend/pipeline.py b/vidmap/frontend/pipeline.py index 8befe1d..805139e 100644 --- a/vidmap/frontend/pipeline.py +++ b/vidmap/frontend/pipeline.py @@ -5,15 +5,13 @@ from pathlib import Path from typing import TYPE_CHECKING, Any -import h5py - -from vidmap.datasets.base import DatasetParser +from vidmap.datasets.capture import CaptureSnapshot, PreparedCapture, copy_unposed_reconstruction from vidmap.frontend.cache import CacheMetadataMismatch, CompleteArtifactContract, IncrementalArtifactContract from vidmap.frontend.correspondences import ImagePair, immutable_array_mapping, validate_correspondence_alignment from vidmap.frontend.depth_output import publish_full_depth_output -from vidmap.frontend.initial_reconstruction import build_initial_reconstruction from vidmap.frontend.paths import FrontendPaths -from vidmap.frontend.preparation.camera_priors import apply_camera_priors +from vidmap.frontend.preparation.camera_priors import CalibrationResult, apply_calibration +from vidmap.frontend.preparation.cameras import CameraPreparationPlan, prepare_capture, resolve_camera_preparation from vidmap.frontend.preparation.geometric_verification import GeometricVerifier from vidmap.frontend.preparation.mapper_inputs import write_verified_mapper_inputs from vidmap.frontend.preparation.tcorr_filtering import CorrespondenceFilter @@ -40,14 +38,12 @@ class FrontendArtifacts: """Cache identities and expected item plans exposed by tracking frontend.""" - track_pairs: CompleteArtifactContract retrieval_pairs: CompleteArtifactContract sparse_features: IncrementalArtifactContract sparse_matches: IncrementalArtifactContract extended_matches: IncrementalArtifactContract depth: IncrementalArtifactContract full_depth: IncrementalArtifactContract | None = None - geocalib: IncrementalArtifactContract | None = None @result_dataclass(frozen=True) @@ -57,17 +53,17 @@ class TrackingFrontendResult: paths: FrontendPaths artifacts: FrontendArtifacts keyframe_sequence: tuple[str, ...] - track_pairs: tuple[ImagePair, ...] retrieval_pairs: tuple[ImagePair, ...] tcorr: Mapping[ImagePair, Any] lc_masks: Mapping[ImagePair, Any] + capture: PreparedCapture + calibration: CalibrationResult | None = None def __post_init__(self): if not isinstance(self.tcorr, Mapping) or not isinstance(self.lc_masks, Mapping): raise TypeError("tcorr and lc_masks must be mappings") validate_correspondence_alignment(self.tcorr, self.lc_masks) object.__setattr__(self, "keyframe_sequence", tuple(self.keyframe_sequence)) - object.__setattr__(self, "track_pairs", tuple(tuple(pair) for pair in self.track_pairs)) object.__setattr__(self, "retrieval_pairs", tuple(tuple(pair) for pair in self.retrieval_pairs)) object.__setattr__(self, "tcorr", immutable_array_mapping(self.tcorr)) object.__setattr__(self, "lc_masks", immutable_array_mapping(self.lc_masks)) @@ -75,21 +71,18 @@ def __post_init__(self): def validate_tracking_result(result: TrackingFrontendResult, *, geocalib_inference: str | None) -> None: """Reject inconsistent optional artifacts at the tracking boundary.""" - if len(result.keyframe_sequence) < 2 or not result.track_pairs: + if len(result.keyframe_sequence) < 2 or not result.artifacts.sparse_matches.expected_items: raise CacheMetadataMismatch("Frontend requires at least two keyframes and one track pair") if result.artifacts.depth is None: raise CacheMetadataMismatch("Depth frontend is enabled but its certified artifact is unavailable") - values = (result.paths.geocalib_per_image_path, result.paths.geocalib_batch_path, result.artifacts.geocalib) + calibration = result.calibration if geocalib_inference is not None: - path = ( - result.paths.geocalib_batch_path - if geocalib_inference == "selected_batch" - else result.paths.geocalib_per_image_path - ) - if path is None or result.artifacts.geocalib is None: + if calibration is None or calibration.evidence.estimator != "geocalib" or calibration.artifact is None: raise CacheMetadataMismatch("GeoCalib is enabled but its certified artifact is unavailable") - elif any(value is not None for value in values): - raise CacheMetadataMismatch("GeoCalib is disabled but an artifact was exposed") + if calibration.evidence.inference != geocalib_inference: + raise CacheMetadataMismatch("GeoCalib inference disagrees with its calibration result") + elif calibration is not None and calibration.evidence.estimator == "geocalib": + raise CacheMetadataMismatch("GeoCalib is disabled but a calibration result was exposed") class Frontend: @@ -103,8 +96,9 @@ def __init__( sample_name: str, cache_dir: Path, sfm_outputs_dir: Path, - scene_parser: DatasetParser, reference_image_names: Sequence[str], + input_snapshot: CaptureSnapshot, + preparation_plan: CameraPreparationPlan | None = None, deterministic_frontend: bool = False, pre_geom_repro_dir: str | Path | None = None, force_recompute: bool = False, @@ -123,7 +117,6 @@ def __init__( self.device = resolve_device(device) self.options = conf self.preparation = conf.preparation - self.estimate_intrinsics = conf.camera_priors.initialization == "predicted" self.sample_name = str(sample_name) self.cache_dir = Path(cache_dir) self.outputs_dir = Path(sfm_outputs_dir) @@ -134,7 +127,13 @@ def __init__( self.mapper_inputs_dir = ( self.outputs_dir / "mapper_inputs" if mapper_inputs_dir is None else Path(mapper_inputs_dir) ) - self.scene_parser = scene_parser + self.scene_parser = input_snapshot.capture + self.preparation_plan = ( + resolve_camera_preparation(self.scene_parser, conf) if preparation_plan is None else preparation_plan + ) + self.prepared_capture = prepare_capture(self.scene_parser, self.preparation_plan) + self.estimate_intrinsics = self.preparation_plan.camera_priors.initialization == "predicted" + self.input_snapshot = input_snapshot self.reference_image_names = tuple(reference_image_names) self.deterministic = bool(deterministic_frontend) self.repro_dir = Path(pre_geom_repro_dir).expanduser() if pre_geom_repro_dir is not None else None @@ -144,11 +143,10 @@ def __init__( def run(self, *, frontend_identity: Mapping[str, object]) -> MapperInputs: """Run the complete frontend and mapper-input publication workflow.""" - tracking, filtered, verification = self._execute_verified_boundary(pre_geom_db_stop=False) + tracking, verification = self._execute_verified_boundary(pre_geom_db_stop=False) mapper_inputs = write_verified_mapper_inputs( self.mapper_inputs_dir, tracking, - filtered, verification, frontend_identity=frontend_identity, ) @@ -162,7 +160,7 @@ def run(self, *, frontend_identity: Mapping[str, object]) -> MapperInputs: def run_pre_geom(self) -> Path: """Run frontend through deterministic pre-GV database construction.""" - _, _, verification = self._execute_verified_boundary(pre_geom_db_stop=True) + _, verification = self._execute_verified_boundary(pre_geom_db_stop=True) return verification.database_path def _execute_verified_boundary(self, *, pre_geom_db_stop: bool): @@ -170,22 +168,12 @@ def _execute_verified_boundary(self, *, pre_geom_db_stop: bool): # Tracking owns all learned predictions and correspondence artifacts. tracking = self.run_tracking() - # Geometry preparation starts from the source cameras and applies any - # calibrated or learned priors before correspondence filtering and GV. - reconstruction = build_initial_reconstruction( - self.scene_parser, reference_image_names=list(self.reference_image_names) - ) - if self.options.camera_priors.initialization == "predicted": - shared = self.options.camera_priors.inference == "selected_batch" - if self.options.camera_priors.estimator == "da3": - path = tracking.paths.depth_maps_path - else: - path = tracking.paths.geocalib_batch_path if shared else tracking.paths.geocalib_per_image_path - names = ("batch_calibration",) if shared else tracking.keyframe_sequence - with h5py.File(path, "r") as hfile: - apply_camera_priors( - results=[hfile[name] for name in names], shared=shared, reconstruction=reconstruction - ) + reconstruction = tracking.capture.rec + if self.estimate_intrinsics: + reconstruction = copy_unposed_reconstruction(reconstruction, tracking.keyframe_sequence) + if tracking.calibration is None: + raise ValueError("Predicted initialization requires calibration results") + apply_calibration(tracking.calibration, reconstruction) correspondence_filter = CorrespondenceFilter( options=self.preparation.lc, @@ -201,11 +189,10 @@ def _execute_verified_boundary(self, *, pre_geom_db_stop: bool): # Persist the masks aligned with the filtered matches before geometric # verification publishes the finalized mapper database. - sparse_matches_path = tracking.paths.sparse_matches_path - lc_masks_path = sparse_matches_path.with_name(f"lc_masks-{sparse_matches_path.stem}.json") - write_loop_closure_masks(filtered.lc_masks, lc_masks_path) + write_loop_closure_masks(filtered.lc_masks, tracking.paths.lc_masks_path) geometric_verifier = GeometricVerifier( + capture=tracking.capture, options=self.preparation.geom_verif, database_path=self.outputs_dir / ("database_pre_geom.db" if pre_geom_db_stop else "database_complete.db"), replay=self.replay, @@ -214,7 +201,7 @@ def _execute_verified_boundary(self, *, pre_geom_db_stop: bool): pre_geom_db_stop=pre_geom_db_stop, ) verification = geometric_verifier.verify(tracking, reconstruction, filtered.tcorr) - return tracking, filtered, verification + return tracking, verification def run_tracking(self) -> TrackingFrontendResult: """Run tracking, depth, and optional GeoCalib stages.""" @@ -225,7 +212,9 @@ def run_tracking(self) -> TrackingFrontendResult: sample_name=self.sample_name, cache_dir=self.cache_dir, cache_namespace=self.frontend_tag if self.namespace_cache_by_config else None, - scene_parser=self.scene_parser, + input_snapshot=self.input_snapshot, + preparation_plan=self.preparation_plan, + prepared_capture=self.prepared_capture, force_recompute=self.force_recompute, cache_full_depth_maps=self.cache_full_depth_maps, deterministic=self.deterministic, diff --git a/vidmap/frontend/preparation/camera_priors.py b/vidmap/frontend/preparation/camera_priors.py index 7e2a0dd..26fd2d4 100644 --- a/vidmap/frontend/preparation/camera_priors.py +++ b/vidmap/frontend/preparation/camera_priors.py @@ -2,9 +2,64 @@ from __future__ import annotations +from dataclasses import dataclass +from pathlib import Path + +import h5py import numpy as np import pycolmap +from vidmap.calibration import CalibrationEvidence, CalibrationObservation +from vidmap.frontend.cache.metadata import IncrementalArtifactContract + + +@dataclass(frozen=True) +class CalibrationResult: + """Certified original predictions and their working-camera associations.""" + + path: Path + artifact: IncrementalArtifactContract + evidence: CalibrationEvidence + + @classmethod + def from_artifact(cls, *, path, reconstruction, image_names, options, artifact): + images = {image.name: image for image in reconstruction.images.values()} + shared = options.inference == "selected_batch" + if shared: + camera_ids = {images[name].camera_id for name in image_names} + if len(camera_ids) != 1: + raise ValueError("Selected shared calibration requires one camera") + observations = (CalibrationObservation(camera_ids.pop(), "batch_calibration", None),) + else: + observations = tuple(CalibrationObservation(images[name].camera_id, name, name) for name in image_names) + evidence = CalibrationEvidence( + options.estimator, + options.inference, + "PINHOLE", + "working_images", + observations, + ) + return cls(Path(path), artifact, evidence) + + +def apply_calibration(result: CalibrationResult, reconstruction: pycolmap.Reconstruction) -> None: + """Initialize from original predictions without changing the stored targets.""" + evidence = result.evidence + images = {image.name: image.camera_id for image in reconstruction.images.values()} + if {observation.camera_id for observation in evidence.observations} != set(images.values()): + raise ValueError("Calibration observations disagree with initialization cameras") + if ( + evidence.inference == "per_view" + and {observation.image_name: observation.camera_id for observation in evidence.observations} != images + ): + raise ValueError("Calibration observations disagree with initialization image membership") + with h5py.File(result.path, "r") as hfile: + apply_camera_priors( + results=[hfile[observation.key] for observation in evidence.observations], + shared=evidence.inference == "selected_batch", + reconstruction=reconstruction, + ) + def _validate_intrinsics(values, *, label: str, positive: bool) -> np.ndarray: values = np.asarray(values, dtype=np.float64) @@ -60,7 +115,7 @@ def apply_camera_priors(*, results, shared, reconstruction): raise ValueError("Predicted initialization requires calibration results") cameras = {image.camera_id: reconstruction.cameras[image.camera_id] for image in reconstruction.images.values()} if any(c.model.name not in {"PINHOLE", "SIMPLE_PINHOLE"} for c in cameras.values()): - raise ValueError("Calibration initialization requires pinhole cameras") + raise ValueError("Unsupported camera model for calibration initialization") if len({(c.model.name, c.width, c.height) for c in cameras.values()}) != 1: raise ValueError("Shared initialization requires identical camera models and dimensions") camera = next(iter(cameras.values())) diff --git a/vidmap/frontend/preparation/cameras.py b/vidmap/frontend/preparation/cameras.py new file mode 100644 index 0000000..6c8d329 --- /dev/null +++ b/vidmap/frontend/preparation/cameras.py @@ -0,0 +1,122 @@ +"""Resolve camera processing and allocate working COLMAP topology once.""" + +from dataclasses import dataclass, replace +from typing import Literal + +import pycolmap + +from vidmap.datasets.capture import PreparedCapture +from vidmap.frontend.options.preparation import CameraPriorEstimationOptions, CameraProcessingOptions +from vidmap.frontend.rig import resolve_rig_options + + +@dataclass(frozen=True) +class CameraPreparationPlan: + allocation: Literal["input", "shared", "per_image"] + camera_priors: CameraPriorEstimationOptions + undistort_camera_ids: tuple[int, ...] + + +def resolve_camera_preparation(capture: PreparedCapture, options) -> CameraPreparationPlan: + """Resolve complete camera settings before prediction or image rendering.""" + if options.rig is not None: + resolve_rig_options(capture, options.rig) + unknown = set(options.camera_processing) - set(capture.rec.cameras) + if unknown: + raise ValueError(f"Camera processing references unknown camera IDs: {sorted(unknown)}") + default = CameraProcessingOptions(camera_priors=options.camera_priors) + settings = { + cid: options.camera_processing[cid] if cid in options.camera_processing else default + for cid in capture.rec.cameras + } + priors = next(iter(settings.values())).camera_priors + if any(setting.camera_priors != priors for setting in settings.values()): + raise ValueError("Different per-camera calibration settings are not supported") + if priors.initialization != options.camera_priors.initialization: + raise ValueError("Camera processing cannot change the input calibration initialization mode") + has_multi_sensor_rig = any(rig.num_sensors() != 1 for rig in capture.rec.rigs.values()) + if capture.preserve_calibration: + if priors.initialization != "supplied": + raise ValueError("Preserved capture calibration requires supplied initialization") + if options.keyframes.selection.intrinsics_source != "ground_truth": + raise ValueError("Preserved capture calibration requires supplied keyframing intrinsics") + allocation = "input" + else: + if has_multi_sensor_rig: + raise ValueError("Camera regrouping requires single-sensor, single-image frames") + allocation = "shared" if priors.initialization == "predicted" else "per_image" + if priors.inference == "selected_batch": + num_cameras = 1 + if allocation == "input": + num_cameras = len(capture.rec.cameras) + elif allocation == "per_image": + num_cameras = capture.rec.num_images() + if num_cameras != 1: + raise ValueError("Selected-batch calibration requires one working camera; select per_view inference") + + undistort = [] + for cid, camera in capture.rec.cameras.items(): + distorted = camera.model_name not in ("PINHOLE", "SIMPLE_PINHOLE") + image_mode = settings[cid].image_mode + if image_mode == "auto": + image_mode = "undistort" if capture.preserve_calibration and distorted else "native" + if image_mode == "undistort" and distorted: + if not capture.preserve_calibration: + raise ValueError("Undistortion requires supplied capture calibration") + undistort.append(cid) + elif distorted: + if capture.preserve_calibration: + raise ValueError("Native calibrated distorted cameras are unsupported; select undistort") + raise ValueError("Native distorted source cameras are unsupported") + return CameraPreparationPlan(allocation, priors, tuple(undistort)) + + +def prepare_capture(capture: PreparedCapture, plan: CameraPreparationPlan) -> PreparedCapture: + """Allocate stable working camera IDs before frame selection or prediction.""" + source = capture.rec + if any(source.camera(cid).model_name not in ("PINHOLE", "SIMPLE_PINHOLE") for cid in plan.undistort_camera_ids): + raise ValueError("Undistort capture images before constructing the frontend") + if plan.allocation == "input": + return capture + rec = pycolmap.Reconstruction() + for index, image in enumerate(sorted(source.images.values(), key=lambda image: image.name)): + cid = 0 if plan.allocation == "shared" else index + camera = source.camera(image.camera_id) + if cid not in rec.cameras: + rec.add_camera_with_trivial_rig( + pycolmap.Camera( + camera_id=cid, + model=camera.model, + width=camera.width, + height=camera.height, + params=camera.params, + has_prior_focal_length=plan.camera_priors.initialization == "supplied", + ) + ) + prepared = pycolmap.Image(image_id=image.image_id, name=image.name, camera_id=cid, frame_id=image.frame_id) + frame = pycolmap.Frame(frame_id=image.frame_id, rig_id=cid) + frame.add_data_id(prepared.data_id) + rec.add_frame(frame) + rec.add_image(prepared) + return replace(capture, rec=rec) + + +def initialize_selected_camera_geometry(working_rec, source_rec, image_names, plan: CameraPreparationPlan): + """Resolve shared image dimensions from the selected images, without regrouping.""" + if plan.allocation != "shared": + return + names = set(image_names) + images = sorted( + (image for image in source_rec.images.values() if image.name in names), key=lambda image: image.name + ) + camera = source_rec.camera(images[0].camera_id) + signature = (camera.model_name, camera.width, camera.height) + if any( + (other.model_name, other.width, other.height) != signature + for other in (source_rec.camera(image.camera_id) for image in images) + ): + raise ValueError("Shared preparation requires identical camera models and dimensions") + prepared = pycolmap.Camera(camera.todict()) + prepared.camera_id = 0 + prepared.has_prior_focal_length = plan.camera_priors.initialization == "supplied" + working_rec.cameras[0] = prepared diff --git a/vidmap/frontend/preparation/geometric_verification.py b/vidmap/frontend/preparation/geometric_verification.py index 76da3ad..7c96891 100644 --- a/vidmap/frontend/preparation/geometric_verification.py +++ b/vidmap/frontend/preparation/geometric_verification.py @@ -1,10 +1,4 @@ -"""Two-stage VGC geometric verification. - -Creates the per-run database (``database_complete.db``), runs geometric -verification either in a single pass or in two passes (strict for VGC and -relaxed for the rest of the pipeline). - -""" +"""Verify the selected image pairs in one caller-owned private database.""" from __future__ import annotations @@ -22,7 +16,8 @@ from vidmap.frontend.colmap_database import create_database_from_frontend, run_geometric_verification from vidmap.frontend.correspondences import ImagePair from vidmap.frontend.options.preparation import GeomVerifOptions -from vidmap.mapper.inputs import FileProvenance, require_finalized_sqlite, sqlite_sidecar_paths +from vidmap.frontend.preparation.mapper_inputs import prepare_frame_sequence +from vidmap.mapper.inputs import require_finalized_sqlite, sqlite_sidecar_paths from vidmap.mapper.replay.evidence.stages import database_file_summary from vidmap.repro.frontend import write_sqlite_summary_artifact from vidmap.utils.logging import progress_bars_enabled @@ -37,11 +32,10 @@ @dataclass(frozen=True) class GeometricVerificationResult: - """Exact finalized database path and optional strict-VGC exclusions.""" + """The published database and its allocated capture timeline.""" database_path: Path - database_provenance: FileProvenance - vgc_filtered_pairs: frozenset[ImagePair] | None + frame_sequence: dict | None def _fsync_file(path: Path) -> None: @@ -116,6 +110,7 @@ def __init__( repro_dir: Path | None, estimate_intrinsics: bool, pre_geom_db_stop: bool = False, + capture, ): self.options = options self.database_path = Path(database_path) @@ -123,6 +118,7 @@ def __init__( self.repro_dir = repro_dir self.estimate_intrinsics = estimate_intrinsics self.pre_geom_db_stop = pre_geom_db_stop + self.capture = capture def verify( self, @@ -141,7 +137,7 @@ def verify( temporary_path = Path(temporary_name) temporary_path.unlink() try: - vgc_filtered_pairs = _verify_database( + allocated = _verify_database( self.options, state, initial_reconstruction, @@ -150,6 +146,7 @@ def verify( database_path=temporary_path, pre_geom_db_stop=self.pre_geom_db_stop, ) + frame_sequence = None if self.pre_geom_db_stop else prepare_frame_sequence(self.capture, allocated) _finalize_sqlite_generation(temporary_path) _prepare_sqlite_destination(self.database_path) os.replace(temporary_path, self.database_path) @@ -171,8 +168,7 @@ def verify( "summary.json", database_file_summary(self.database_path), ) - provenance = FileProvenance.from_path(self.database_path) - return GeometricVerificationResult(self.database_path, provenance, vgc_filtered_pairs) + return GeometricVerificationResult(self.database_path, frame_sequence) def _verify_database( @@ -184,25 +180,16 @@ def _verify_database( estimate_intrinsics: bool, database_path: Path, pre_geom_db_stop: bool, -) -> frozenset[ImagePair] | None: - if pre_geom_db_stop: - create_database_from_frontend( - state, - initial_reconstruction, - database_path, - estimate_intrinsics=estimate_intrinsics, - matches=tcorr, - ) - - return None - - create_database_from_frontend( +) -> pycolmap.Reconstruction: + allocated = create_database_from_frontend( state, initial_reconstruction, database_path, estimate_intrinsics=estimate_intrinsics, matches=tcorr, ) + if pre_geom_db_stop: + return allocated geom_verif_options = _verification_options(options, options.max_H_inlier_ratio) run_geometric_verification( @@ -211,4 +198,4 @@ def _verify_database( progress_bars_enabled(), geom_verif_options, ) - return None + return allocated diff --git a/vidmap/frontend/preparation/mapper_inputs.py b/vidmap/frontend/preparation/mapper_inputs.py index df15a17..9f16721 100644 --- a/vidmap/frontend/preparation/mapper_inputs.py +++ b/vidmap/frontend/preparation/mapper_inputs.py @@ -8,7 +8,8 @@ from pathlib import Path from typing import TYPE_CHECKING, AbstractSet -from vidmap.frontend.cache import CacheMetadataMismatch, read_pair_artifact, validate_incremental_cache +from vidmap.calibration import CalibrationEvidence +from vidmap.frontend.cache import CacheMetadataMismatch, validate_incremental_cache from vidmap.mapper.inputs import ( CANONICAL_NAMES, DATABASE_NAME, @@ -16,7 +17,6 @@ LC_MASKS_NAME, MANIFEST_NAME, REQUIRED_PAYLOAD_NAMES, - TRACK_PAIRS_NAME, VGC_FILTERED_PAIRS_NAME, MapperInputs, require_finalized_sqlite, @@ -28,21 +28,22 @@ if TYPE_CHECKING: from vidmap.frontend.pipeline import TrackingFrontendResult from vidmap.frontend.preparation.geometric_verification import GeometricVerificationResult - from vidmap.frontend.preparation.tcorr_filtering import FilteredCorrespondences def write_mapper_inputs( directory: Path, sources: Mapping[str, Path], lc_masks: Mapping[tuple[str, str], object] | None, - vgc_filtered_pairs: AbstractSet[tuple[str, str]] | None, + vgc_filtered_pairs: AbstractSet[tuple[str, str]] | None = None, *, frontend_identity: Mapping[str, object], + frame_sequence: dict, + calibration: CalibrationEvidence | None = None, ) -> MapperInputs: """Replace the directory with one self-contained mapper-input generation.""" directory = resolve_mapper_inputs_directory(directory) source_names = set(sources) - generated_names = {LC_MASKS_NAME, VGC_FILTERED_PAIRS_NAME, MANIFEST_NAME} + generated_names = {VGC_FILTERED_PAIRS_NAME, MANIFEST_NAME} unexpected = source_names - (CANONICAL_NAMES - generated_names) if unexpected: raise ValueError(f"Unexpected mapper-input names: {sorted(unexpected)}") @@ -59,6 +60,8 @@ def write_mapper_inputs( if name == DATABASE_NAME: require_finalized_sqlite(source) + if lc_masks is not None and LC_MASKS_NAME in source_paths: + raise ValueError("Loop-closure masks must have exactly one source") if directory.exists(): shutil.rmtree(directory) directory.mkdir(parents=True) @@ -75,14 +78,20 @@ def write_mapper_inputs( ) + "\n" ) - write_mapper_inputs_manifest(directory, frontend_identity=frontend_identity) - return MapperInputs.from_directory(directory, expected_identity=frontend_identity) + write_mapper_inputs_manifest( + directory, + frontend_identity=frontend_identity, + frame_sequence=frame_sequence, + calibration=calibration, + ) + result = MapperInputs.from_directory(directory, expected_identity=frontend_identity) + result.validate() + return result def write_verified_mapper_inputs( directory: Path, state: TrackingFrontendResult, - filtered: FilteredCorrespondences, verification: GeometricVerificationResult, *, frontend_identity: Mapping[str, object], @@ -93,13 +102,6 @@ def write_verified_mapper_inputs( if depth_contract is None: raise CacheMetadataMismatch("Depth mapper input has no frontend cache contract") - verification.database_provenance.validate(verification.database_path) - track_pairs = read_pair_artifact( - paths.track_pairs_path, - state.artifacts.track_pairs.metadata, - ) - if tuple(track_pairs) != state.track_pairs: - raise CacheMetadataMismatch("Track-pair cache changed before mapper-input publication") validate_incremental_cache( paths.depth_maps_path, depth_contract.metadata, @@ -108,27 +110,34 @@ def write_verified_mapper_inputs( sources = { DATABASE_NAME: verification.database_path, - TRACK_PAIRS_NAME: paths.track_pairs_path, + LC_MASKS_NAME: paths.lc_masks_path, DEPTH_MAPS_NAME: paths.depth_maps_path, } - if state.artifacts.geocalib is not None: + calibration = state.calibration + if calibration is not None and calibration.evidence.estimator == "geocalib": from vidmap.mapper.inputs.snapshot import calibration_artifact_name - contract = state.artifacts.geocalib - calibration = frontend_identity["boundary_options"] - filename = calibration_artifact_name(calibration["estimator"], calibration["inference"]) - path = ( - paths.geocalib_batch_path - if calibration["inference"] == "selected_batch" - else paths.geocalib_per_image_path - ) - validate_incremental_cache(path, contract.metadata, contract.expected_items) - sources[filename] = path + contract = calibration.artifact + validate_incremental_cache(calibration.path, contract.metadata, contract.expected_items) + filename = calibration_artifact_name(calibration.evidence.estimator, calibration.evidence.inference) + sources[filename] = calibration.path return write_mapper_inputs( directory, sources, - filtered.lc_masks, - verification.vgc_filtered_pairs, + None, frontend_identity=frontend_identity, + frame_sequence=verification.frame_sequence, + calibration=None if calibration is None else calibration.evidence, ) + + +def prepare_frame_sequence(capture, allocated): + """Carry allocated frame IDs forward in capture order.""" + + frames = allocated.frames + ids = [fid for fid in capture.frame_ids if fid in frames] + if set(ids) != set(frames): + raise ValueError("Allocated frame inventory disagrees with selected capture") + times = None if capture.frame_timestamps_ns is None else [capture.frame_timestamps_ns[fid] for fid in ids] + return {"frame_ids": ids, "timestamps_ns": times} diff --git a/vidmap/frontend/rig.py b/vidmap/frontend/rig.py new file mode 100644 index 0000000..9b1bee1 --- /dev/null +++ b/vidmap/frontend/rig.py @@ -0,0 +1,54 @@ +"""Frontend selection and pair policy over authoritative COLMAP membership.""" + +from vidmap.datasets.capture import PreparedCapture +from vidmap.frontend.options.rig import RigOptions + + +def resolve_rig_options(capture: PreparedCapture, options: RigOptions): + """Resolve the admission sensor and ordered same-frame matching policy.""" + reference = capture.reference_camera_id if options.keyframe_camera is None else options.keyframe_camera + if reference not in capture.rec.cameras: + raise ValueError("The keyframing camera must belong to the capture") + if any(cid not in capture.rec.cameras for pair in options.match_same_timestamp for cid in pair): + raise ValueError("Same-timestamp matching references an unknown camera") + for first, second in options.match_same_timestamp: + if not any( + rig.has_sensor(capture.rec.camera(first).sensor_id) + and rig.has_sensor(capture.rec.camera(second).sensor_id) + for rig in capture.rec.rigs.values() + ): + raise ValueError("Same-timestamp camera pairs must belong to the same rig") + sequential = {frozenset(pair) for pair in options.flag_same_timestamp_as_sequential} + return reference, tuple( + (first, second, frozenset((first, second)) in sequential) + for first, second in sorted(tuple(sorted(pair)) for pair in options.match_same_timestamp) + ) + + +def camera_streams(capture: PreparedCapture, names=None, reference_camera_id=None): + """Derive temporal streams from native frames, without copying their graph.""" + selected = None if names is None else set(names) + if selected is not None and ( + len(selected) != len(names) or not selected <= {im.name for im in capture.rec.images.values()} + ): + raise ValueError("Selected capture images must be unique and belong to the capture") + if names is not None and reference_camera_id is None: + cameras = tuple(dict.fromkeys(capture.image(name).camera_id for name in names)) + else: + reference = capture.reference_camera_id if reference_camera_id is None else reference_camera_id + if reference not in capture.rec.cameras: + raise ValueError("The reference camera must belong to the capture") + cameras = (reference,) + cameras = (*cameras, *sorted(set(capture.rec.cameras) - set(cameras))) + camera_order = {cid: index for index, cid in enumerate(cameras)} + streams = [] + for fid in capture.frame_ids: + frame = capture.rec.frame(fid) + members = [capture.rec.image(data.id) for data in frame.image_ids] + members.sort(key=lambda image: camera_order[image.camera_id]) + if not streams: + streams = [[] for _ in members] + for stream, image in zip(streams, members, strict=True): + if selected is None or image.name in selected: + stream.append(image.name) + return tuple(tuple(stream) for stream in streams if stream) diff --git a/vidmap/frontend/runner.py b/vidmap/frontend/runner.py index 3f08cf8..e059d9a 100644 --- a/vidmap/frontend/runner.py +++ b/vidmap/frontend/runner.py @@ -4,7 +4,7 @@ import logging import shutil -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path from typing import TYPE_CHECKING @@ -12,6 +12,7 @@ from vidmap.benchmark.results import FrontendResultStore from vidmap.frontend.identity import FrontendIdentity from vidmap.frontend.pipeline import Frontend +from vidmap.frontend.preparation.cameras import prepare_capture, resolve_camera_preparation from vidmap.mapper.inputs import MapperInputs from vidmap.utils.device import resolve_device from vidmap.utils.logging import log_context @@ -19,6 +20,8 @@ if TYPE_CHECKING: import torch + from vidmap.datasets.capture import PreparedCapture + logger = logging.getLogger(__name__) @@ -49,37 +52,74 @@ class FrontendRunResult: def run_local_frontend( conf, - input_path: str | Path, + input_path: str | Path | PreparedCapture, *, workspace: str | Path, imnames=None, - intrinsics_path: str | Path | None = None, force_frontend: bool = False, cache_depth_maps: bool = False, mapper_inputs_path: str | Path | None = None, device: str | torch.device | None = None, ) -> FrontendTargetResult: - """Process one concrete image directory or MP4 into a finalized boundary.""" + """Process local media or a prepared capture into a finalized boundary.""" + from vidmap.datasets.capture import PreparedCapture, prepare_scene_capture from vidmap.datasets.local import LocalImageParser - from vidmap.reconstruction import prepare_reconstruction_images, write_local_input_provenance - device = resolve_device(device) workspace = Path(workspace).expanduser() - image_dir = prepare_reconstruction_images(input_path, workspace) - scene_parser = LocalImageParser( - image_dir=image_dir, - imnames=imnames, - intrinsics_path=intrinsics_path, - estimate_intrinsics=conf.pipeline.camera_priors.initialization == "predicted", - ) + input_snapshot = None + preparation_plan = None + if isinstance(input_path, PreparedCapture): + if imnames is not None: + raise ValueError("Capture inputs already supply calibration and complete frames") + scene_parser = input_path + elif conf.pipeline.rig is not None: + if imnames is not None: + raise ValueError("Rig configuration supplies calibration and complete frames") + rig = conf.pipeline.rig + from vidmap.datasets.prepare.rig_capture import load_capture, prepare_rig_capture + + source_capture = load_capture(input_path, rig) + preparation_plan = resolve_camera_preparation(source_capture, conf.pipeline) + input_snapshot = prepare_rig_capture( + source_capture, + workspace / "prepared_capture", + camera_ids_to_undistort=preparation_plan.undistort_camera_ids, + ) + scene_parser = input_snapshot.capture + else: + from vidmap.reconstruction import prepare_reconstruction_images + + if conf.pipeline.camera_priors.initialization != "predicted": + raise ValueError("Calibrated local input requires a rig configuration, including for one camera") + image_dir = prepare_reconstruction_images(input_path, workspace) + scene_parser = LocalImageParser(image_dir=image_dir, imnames=imnames) + from vidmap.reconstruction import write_local_input_provenance + + scene_parser = prepare_scene_capture(scene_parser) + if preparation_plan is None: + preparation_plan = resolve_camera_preparation(scene_parser, conf.pipeline) + if preparation_plan.undistort_camera_ids: + from vidmap.datasets.prepare.rig_capture import prepare_rig_capture + + input_snapshot = prepare_rig_capture( + scene_parser, + workspace / "prepared_capture", + camera_ids_to_undistort=preparation_plan.undistort_camera_ids, + ) + scene_parser = input_snapshot.capture + if input_snapshot is None and (scene_parser.kind is not None or scene_parser.frame_timestamps_ns is not None): + input_snapshot = scene_parser.snapshot() + device = resolve_device(device) + image_dir = scene_parser.rgb_dir reference_image_ids = tuple(scene_parser.rec.images) identity = FrontendIdentity.from_config( conf, dataset="local", - scene="local", + scene="local" if scene_parser.kind is None else scene_parser.scene, mode="all", testset_id="input", reference_image_ids=reference_image_ids, + source_fingerprint=None if input_snapshot is None else input_snapshot.fingerprint(), ) temporary_dir = workspace / "tmp_cache_dir" mapper_inputs_dir = ( @@ -92,11 +132,16 @@ def run_local_frontend( expected_identity=identity.as_dict(), ) mapper_inputs.validate() - write_local_input_provenance(mapper_inputs.directory, image_dir, overwrite=True) + write_local_input_provenance( + mapper_inputs.directory, + image_dir, + overwrite=True, + ) if cache_depth_maps: import h5py from vidmap.frontend.depth import cache_full_depth_maps_posthoc + from vidmap.frontend.image_dataset import FrameSequence selected = set() with h5py.File(mapper_inputs.depth_maps_path, "r") as hfile: @@ -107,12 +152,13 @@ def collect_depth_group(name, item): hfile.visititems(collect_depth_group) selected = frozenset(selected) - image_names = tuple(name for name in scene_parser.imnames if name in selected) + frames = FrameSequence.from_scene(scene_parser, rig_options=conf.pipeline.rig) + image_names = tuple(name for name in frames.names if name in selected) if len(image_names) != len(selected): missing = sorted(selected - set(image_names)) raise ValueError(f"Mapper-input depth names are absent from local media: {missing}") cache_full_depth_maps_posthoc( - scene_parser=scene_parser, + scene_parser=prepare_capture(scene_parser, preparation_plan), image_names=image_names, sampled_depth_path=mapper_inputs.depth_maps_path, output_path=full_depth_maps_path, @@ -129,6 +175,9 @@ def collect_depth_group(name, item): return FrontendTargetResult(identity, mapper_inputs, reused=True) if mapper_inputs_path is not None: raise FileNotFoundError(f"Finalized mapper inputs not found: {mapper_inputs_dir}") + if input_snapshot is None: + input_snapshot = scene_parser.snapshot() + identity = replace(identity, source_fingerprint=input_snapshot.fingerprint()) frontend = Frontend( conf=conf.pipeline, replay_cache=conf.run.replay_cache, @@ -137,7 +186,8 @@ def collect_depth_group(name, item): pre_geom_repro_dir=conf.run.pre_geom_repro_dir, cache_dir=temporary_dir / "cache", sfm_outputs_dir=temporary_dir / "work", - scene_parser=scene_parser, + input_snapshot=input_snapshot, + preparation_plan=preparation_plan, reference_image_names=scene_parser.imnames, force_recompute=force_frontend, cache_full_depth_maps=cache_depth_maps, @@ -148,7 +198,11 @@ def collect_depth_group(name, item): ) mapper_inputs = frontend.run(frontend_identity=identity.as_dict()) mapper_inputs.validate() - write_local_input_provenance(mapper_inputs.directory, image_dir, overwrite=True) + write_local_input_provenance( + mapper_inputs.directory, + image_dir, + overwrite=True, + ) if temporary_dir.exists(): shutil.rmtree(temporary_dir) return FrontendTargetResult(identity, mapper_inputs, reused=False) @@ -163,6 +217,7 @@ def __init__(self, conf, selection, run_options=None, *, output_root=None) -> No def run_with_results(self, *, force_frontend: bool = False) -> FrontendRunResult: """Run frontend and return every concrete finalized boundary.""" + from vidmap.datasets.capture import prepare_scene_capture from vidmap.reconstruction import write_local_input_provenance failure_count = 0 @@ -172,14 +227,6 @@ def run_with_results(self, *, force_frontend: bool = False) -> FrontendRunResult scene_parser, reference_image_names, ) in self.selection.prepared_targets(): - identity = FrontendIdentity.from_config( - self.conf, - dataset=self.selection.dataset_spec.name, - scene=target.scene, - mode=target.testset_type, - testset_id=target.testset_desc, - reference_image_ids=target.ref_imids, - ) results = FrontendResultStore( self.conf, self.selection.dataset_layout, @@ -196,6 +243,21 @@ def run_with_results(self, *, force_frontend: bool = False) -> FrontendRunResult config=self.conf.name, ): try: + scene_parser = prepare_scene_capture(scene_parser) + input_snapshot = ( + scene_parser.snapshot() + if scene_parser.kind is not None or scene_parser.frame_timestamps_ns is not None + else None + ) + identity = FrontendIdentity.from_config( + self.conf, + dataset=self.selection.dataset_spec.name, + scene=target.scene, + mode=target.testset_type, + testset_id=target.testset_desc, + reference_image_ids=target.ref_imids, + source_fingerprint=None if input_snapshot is None else input_snapshot.fingerprint(), + ) existing = ( None if force_frontend @@ -203,7 +265,11 @@ def run_with_results(self, *, force_frontend: bool = False) -> FrontendRunResult ) if existing is not None: existing.validate() - write_local_input_provenance(existing.directory, scene_parser.rgb_dir, overwrite=True) + write_local_input_provenance( + existing.directory, + scene_parser.rgb_dir, + overwrite=True, + ) result = FrontendTargetResult(identity, existing, reused=True) completed.append(result) logger.info( @@ -212,6 +278,9 @@ def run_with_results(self, *, force_frontend: bool = False) -> FrontendRunResult result.path, ) continue + if input_snapshot is None: + input_snapshot = scene_parser.snapshot() + identity = replace(identity, source_fingerprint=input_snapshot.fingerprint()) results.cache_dir.mkdir(parents=True, exist_ok=True) frontend = Frontend( conf=self.conf.pipeline, @@ -221,7 +290,7 @@ def run_with_results(self, *, force_frontend: bool = False) -> FrontendRunResult pre_geom_repro_dir=self.conf.run.pre_geom_repro_dir, cache_dir=results.cache_dir, sfm_outputs_dir=results.resolved_frontend_work_dir(), - scene_parser=scene_parser, + input_snapshot=input_snapshot, reference_image_names=reference_image_names, force_recompute=force_frontend, frontend_tag=identity.tag, @@ -233,7 +302,11 @@ def run_with_results(self, *, force_frontend: bool = False) -> FrontendRunResult frontend.run_pre_geom() else: mapper_inputs = frontend.run(frontend_identity=identity.as_dict()) - write_local_input_provenance(mapper_inputs.directory, scene_parser.rgb_dir, overwrite=True) + write_local_input_provenance( + mapper_inputs.directory, + scene_parser.rgb_dir, + overwrite=True, + ) result = FrontendTargetResult(identity, mapper_inputs, reused=False) completed.append(result) logger.info( diff --git a/vidmap/frontend/tracking/composition.py b/vidmap/frontend/tracking/composition.py index 273825e..6ce6b4a 100644 --- a/vidmap/frontend/tracking/composition.py +++ b/vidmap/frontend/tracking/composition.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: import torch -from vidmap.frontend.cache import fingerprint, ordered_files_fingerprint +from vidmap.frontend.cache import fingerprint from vidmap.frontend.depth import DepthEstimator from vidmap.frontend.geocalib import CameraPriorEstimator from vidmap.frontend.image_dataset import FrameSequence @@ -19,6 +19,8 @@ from vidmap.frontend.loop_closure.extended_matches import ExtendedMatchBuilder from vidmap.frontend.models.romav2 import create_lazy_romav2_tracker, romav2_cache_identity from vidmap.frontend.paths import build_frontend_paths +from vidmap.frontend.preparation.camera_priors import CalibrationResult +from vidmap.frontend.preparation.cameras import initialize_selected_camera_geometry from vidmap.frontend.tracking.sparse_tracks import SparseTrackBuilder from vidmap.frontend.tracking.transitive import TransitiveCorrespondenceBuilder @@ -35,7 +37,9 @@ def __init__( sample_name, cache_dir, cache_namespace, - scene_parser, + input_snapshot, + preparation_plan, + prepared_capture, force_recompute, deterministic, repro_dir, @@ -47,7 +51,9 @@ def __init__( self.sample_name = sample_name self.cache_dir = Path(cache_dir) self.cache_namespace = None if cache_namespace is None else str(cache_namespace) - self.scene_parser = scene_parser + self.input_snapshot = input_snapshot + self.preparation_plan = preparation_plan + self.prepared_capture = prepared_capture self.force_recompute = force_recompute self.cache_full_depth_maps = bool(cache_full_depth_maps) self.deterministic = deterministic @@ -61,9 +67,12 @@ def run(self): def _run(self, preparation): from vidmap.frontend.pipeline import FrontendArtifacts, TrackingFrontendResult, validate_tracking_result + source = self.input_snapshot.capture + working_capture = self.prepared_capture options = self.options tracker_options = options.roma - use_geocalib = options.camera_priors.estimator == "geocalib" + prior_options = self.preparation_plan.camera_priors + use_geocalib = prior_options.estimator == "geocalib" # Resolve every cache path from the frontend identity so all stages # operate on one immutable artifact plan. @@ -76,18 +85,19 @@ def _run(self, preparation): config_name=self.cache_namespace, cache_variant="cache-" + fingerprint({"tracker": romav2_cache_identity(tracker_options)})[:16], ) - paths.track_pairs_path.parent.mkdir(exist_ok=True, parents=True) + paths.admitted_frames_path.parent.mkdir(exist_ok=True, parents=True) logger.info("Input sparse features: %s", paths.sparse_features_path) logger.info("Depth maps: %s", paths.depth_maps_path) if use_geocalib: logger.info("Geo-calibration (per-image): %s", paths.geocalib_per_image_path) logger.info("Geo-calibration (batch): %s", paths.geocalib_batch_path) - frames = FrameSequence.from_scene(self.scene_parser) - + frames = FrameSequence.from_scene(source, rig_options=options.rig) + geocalib = None + calibration = None verification = None with create_lazy_romav2_tracker(tracker_options, device=self.device) as tracker: keyframe_processor = KeyframeProcessor( - scene_parser=self.scene_parser, + input_snapshot=self.input_snapshot, frames=frames, paths=paths, force_recompute=self.force_recompute, @@ -98,19 +108,20 @@ def _run(self, preparation): salient_options=options.keyframes.features, device=self.device, ) - track_pairs_metadata = keyframe_cache.admitted_track_pairs_cache_metadata( - scene_parser=self.scene_parser, - sequence=frames.names, - timestamps=frames.timestamps, + admission_metadata = keyframe_cache.admitted_frames_cache_metadata( + input_snapshot=self.input_snapshot, + sequence=frames.streams[0], + frame_ids=frames.frame_ids, tracker_options=tracker_options, lowres_options=options.keyframes.matching, highres_options=options.tracks.images, keyframe_options=options.keyframes.selection, salient_options=options.keyframes.features, + track_options=options.tracks.propagation, ) - keyframes = keyframe_processor.load_cached(track_pairs_metadata) + keyframes = keyframe_processor.load_cached(admission_metadata) sparse_track_builder = SparseTrackBuilder( - scene_parser=self.scene_parser, + scene_parser=source, paths=paths, force_recompute=self.force_recompute, repro_dir=self.repro_dir, @@ -123,13 +134,12 @@ def _run(self, preparation): extended_options=options.loop_closure, device=self.device, ) + if keyframes is not None: + keyframe_processor.complete_salient_features(keyframes) tracks = sparse_track_builder.load_complete() if tracks is None: provisional_salient_metadata = keyframe_cache.salient_feature_cache_metadata( salient_options=options.keyframes.features, - sequence=frames.names, - timestamps=frames.timestamps, - track_pairs_metadata=track_pairs_metadata, ) candidates = keyframe_processor.select_candidates(provisional_salient_metadata) from vidmap.frontend.models.depth.da3_video import verify_da3_model_snapshot @@ -141,12 +151,13 @@ def _run(self, preparation): lambda admitted_ids: keyframe_processor.commit_keyframes( admitted_ids, candidates.forced_indices, - track_pairs_metadata, + admission_metadata, ), + complete_salients=keyframe_processor.complete_salient_features, ) transitive_builder = TransitiveCorrespondenceBuilder( - scene_parser=self.scene_parser, + scene_parser=source, paths=paths, repro_dir=self.repro_dir, keyframes=keyframes, @@ -154,9 +165,10 @@ def _run(self, preparation): ) transitive = transitive_builder.build() - image_content = ordered_files_fingerprint(self.scene_parser.rgb_dir, keyframes.names) + image_content = self.input_snapshot.image_content_fingerprint(keyframes.names) extended_match_builder = ExtendedMatchBuilder( - scene_parser=self.scene_parser, + keyframes=keyframes, + scene_parser=source, paths=paths, force_recompute=self.force_recompute, repro_dir=self.repro_dir, @@ -175,12 +187,12 @@ def _run(self, preparation): if verification is not None: verification.result() depth_estimator = DepthEstimator( - scene_parser=self.scene_parser, + scene_parser=source, paths=paths, force_recompute=self.force_recompute, keyframes=keyframes, options=options.depth, - calibration_enabled=options.camera_priors.estimator == "da3", + calibration_enabled=prior_options.estimator == "da3", image_content_fingerprint=image_content, device=self.device, ) @@ -189,40 +201,57 @@ def _run(self, preparation): if self.cache_full_depth_maps else depth_estimator.estimate() ) - geocalib = self._estimate_camera_priors(paths, keyframes.names, image_content) + if use_geocalib: + geocalib = self._camera_prior_estimator(paths, keyframes.names, image_content).estimate() + + initialize_selected_camera_geometry(working_capture.rec, source.rec, keyframes.names, self.preparation_plan) + if prior_options.estimator != "none": + if prior_options.estimator == "da3": + calibration_path, calibration_artifact = paths.depth_maps_path, depth.sampled + else: + calibration_path = ( + paths.geocalib_batch_path + if prior_options.inference == "selected_batch" + else paths.geocalib_per_image_path + ) + calibration_artifact = geocalib + calibration = CalibrationResult.from_artifact( + path=calibration_path, + reconstruction=working_capture.rec, + image_names=keyframes.names, + options=prior_options, + artifact=calibration_artifact, + ) logger.info("Features located in %s", paths.sparse_features_path) result = TrackingFrontendResult( paths=paths, artifacts=FrontendArtifacts( - track_pairs=tracks.track_pairs_artifact, retrieval_pairs=extended.retrieval_pairs_artifact, sparse_features=tracks.sparse_features, sparse_matches=tracks.sparse_matches, extended_matches=extended.artifact, depth=depth.sampled, full_depth=depth.full, - geocalib=geocalib, ), keyframe_sequence=keyframes.names, - track_pairs=tracks.track_pairs, retrieval_pairs=extended.retrieval_pairs, tcorr=extended.tcorr, lc_masks=extended.lc_masks, + capture=working_capture, + calibration=calibration, ) - validate_tracking_result(result, geocalib_inference=options.camera_priors.inference if use_geocalib else None) + validate_tracking_result(result, geocalib_inference=prior_options.inference if use_geocalib else None) return result - def _estimate_camera_priors(self, paths, keyframe_names, image_content): - if self.options.camera_priors.estimator != "geocalib": - return None + def _camera_prior_estimator(self, paths, keyframe_names, image_content): return CameraPriorEstimator( - rgb_dir=self.scene_parser.rgb_dir, + rgb_dir=self.input_snapshot.capture.rgb_dir, per_image_path=paths.geocalib_per_image_path, batch_path=paths.geocalib_batch_path, force_recompute=self.force_recompute, keyframe_names=keyframe_names, - options=self.options.camera_priors, + options=self.preparation_plan.camera_priors, image_content_fingerprint=image_content, device=self.device, - ).estimate() + ) diff --git a/vidmap/frontend/tracking/propagation.py b/vidmap/frontend/tracking/propagation.py index 86a598f..0fd6730 100644 --- a/vidmap/frontend/tracking/propagation.py +++ b/vidmap/frontend/tracking/propagation.py @@ -22,8 +22,8 @@ from vidmap.frontend.h5_write_queue import H5WriteQueue, save_keypoints, write_pair_matches from vidmap.frontend.image_dataset import ImageDatasetOptions, get_image_size from vidmap.frontend.keyframes.selection import score_keyframe_lookahead +from vidmap.frontend.loop_closure.matching import queue_extended_matches from vidmap.frontend.paths import FrontendPaths -from vidmap.frontend.tracking.kernels import select_lc_matches_from_dense from vidmap.frontend.tracking.multiflow import build_multiflow_window from vidmap.frontend.tracking.state import SparseTrackState from vidmap.frontend.video_images import RomaVideoImageDataset, load_roma_resolution_pair @@ -84,28 +84,16 @@ def _write_sequential_match( writer_queue, ): """Convert one reused dense field into a queued sequential sparse match.""" - pair_name = names_to_pair(name0, name1) - kpts0 = get_keypoints(sparse_features_path, name0) - kpts1 = get_keypoints(sparse_features_path, name1) - width0, height0 = get_image_size(scene_parser, name0) - width1, height1 = get_image_size(scene_parser, name1) - lc_matches = select_lc_matches_from_dense( - kpts0, - kpts1, + queue_extended_matches( + name0, + name1, + get_keypoints(sparse_features_path, name0), + get_keypoints(sparse_features_path, name1), dense.matches, dense.certainty, - source_size=(width0, height0), - target_size=(width1, height1), - lc_match_thresh=match_threshold, - ) - writer_queue.put( - ( - pair_name, - { - "matches0": torch.from_numpy(lc_matches["matches0"])[None], - "matching_scores0": torch.from_numpy(lc_matches["matching_scores0"]).float()[None], - }, - ) + scene_parser=scene_parser, + match_threshold=match_threshold, + writer_queue=writer_queue, ) diff --git a/vidmap/frontend/tracking/sparse_tracks.py b/vidmap/frontend/tracking/sparse_tracks.py index fd8a90f..78f2532 100644 --- a/vidmap/frontend/tracking/sparse_tracks.py +++ b/vidmap/frontend/tracking/sparse_tracks.py @@ -5,25 +5,25 @@ import logging import tempfile from dataclasses import dataclass as internal_dataclass +from functools import partial from pathlib import Path from typing import TYPE_CHECKING, Any, Mapping -from vidmap.datasets.base import DatasetParser +from vidmap.datasets.capture import PreparedCapture from vidmap.frontend.cache import ( - CompleteArtifactContract, IncrementalArtifactContract, artifact_fingerprint, cache_metadata, - certify_complete_artifact, certify_incremental_artifact, incremental_cache_is_complete, mark_incremental_cache_complete, read_cache_metadata, semantic_config, ) -from vidmap.frontend.correspondences import validate_pair_name_plan +from vidmap.frontend.correspondences import combine_and_deduplicate_pairs, validate_pair_name_plan from vidmap.frontend.keyframes.processing import KeyframePlan from vidmap.frontend.loop_closure.cache import ExtendedMatchCache +from vidmap.frontend.loop_closure.matching import EXTENDED_MATCH_DIRECTION_POLICY from vidmap.frontend.models.romav2 import LazyRoMaV2Tracker, romav2_cache_identity from vidmap.frontend.options.matching import RoMaImageOptions, RoMaV2Options from vidmap.frontend.options.tracking import SparseTrackOptions @@ -54,7 +54,6 @@ class _TrackCachePlan: keyframe_sequence: tuple[str, ...] keyframe_pairs: tuple[tuple[str, str], ...] sequential_pairs: tuple[tuple[str, str], ...] - track_pairs: Mapping[str, Any] outputs: _TrackOutputMetadata @@ -64,7 +63,6 @@ class SparseTrackResult: track_pairs: tuple[tuple[str, str], ...] sequential_pairs: tuple[tuple[str, str], ...] - track_pairs_artifact: CompleteArtifactContract sparse_features: IncrementalArtifactContract sparse_matches: IncrementalArtifactContract extended_matches: ExtendedMatchCache @@ -85,13 +83,6 @@ def overlap_pair_plan(keyframe_sequence, *, track_options): ) -def _trackprop_cache_config(track_options, lc_match_thresh): - """Return the sparse-track options that affect propagation outputs.""" - config = semantic_config(track_options, exclude_fields=_RUNTIME_CONFIG_FIELDS) - config["lc_match_thresh"] = lc_match_thresh - return config - - def _highres_cache_config(highres_options, lowres_match_resolution): """Return image and low-resolution matching settings used by tracking.""" config = semantic_config(highres_options) @@ -103,16 +94,15 @@ def track_output_cache_metadata( track_options, highres_options, lowres_match_resolution, - extended_options, tracker_options, keyframe_sequence, keyframe_pairs, - track_pairs_metadata, + admission_fingerprint, salient_features_fingerprint, ): common_config = { "tracker": romav2_cache_identity(tracker_options), - "trackprop": _trackprop_cache_config(track_options, extended_options.lc_match_thresh), + "trackprop": semantic_config(track_options, exclude_fields=_RUNTIME_CONFIG_FIELDS), "highres": _highres_cache_config(highres_options, lowres_match_resolution), } ordered_inputs = { @@ -120,7 +110,7 @@ def track_output_cache_metadata( "keyframe_pairs": keyframe_pairs, } upstream = { - "track_pairs": artifact_fingerprint(track_pairs_metadata), + "admission": admission_fingerprint, "salient_features": salient_features_fingerprint, } return _TrackOutputMetadata( @@ -146,33 +136,24 @@ def track_output_cache_metadata( def extended_matches_cache_metadata( - track_options, highres_options, lowres_match_resolution, extended_options, tracker_options, - keyframe_sequence, - sequential_pairs, - track_pairs_metadata, sparse_features_metadata, ): + """Bind finalized inference to its sparse index domain and sampling settings.""" return cache_metadata( stage="extended_matches", config={ "tracker": romav2_cache_identity(tracker_options), "highres": _highres_cache_config(highres_options, lowres_match_resolution), - "trackprop": _trackprop_cache_config(track_options, extended_options.lc_match_thresh), - "retrieval": {}, - "extended": extended_options, - }, - ordered_inputs={ - "keyframe_sequence": keyframe_sequence, - "sequential_pairs": sequential_pairs, - }, - upstream={ - "track_pairs": artifact_fingerprint(track_pairs_metadata), - "sparse_features": artifact_fingerprint(sparse_features_metadata), + "inference_direction": EXTENDED_MATCH_DIRECTION_POLICY, + "retrieval_direction": "temporal" if extended_options.lc_pair_nms else "canonical", + "lc_match_thresh": extended_options.lc_match_thresh, }, + ordered_inputs={}, + upstream={"sparse_features": artifact_fingerprint(sparse_features_metadata)}, payload_format="per-pair-extended-matches", nonsemantic_config_fields=_RUNTIME_CONFIG_FIELDS, ) @@ -184,7 +165,7 @@ class SparseTrackBuilder: def __init__( self, *, - scene_parser: DatasetParser, + scene_parser: PreparedCapture, paths: FrontendPaths, force_recompute: bool, repro_dir: Path | None, @@ -215,32 +196,25 @@ def _cache_plan(self) -> _TrackCachePlan: keyframe_sequence = tuple(self.keyframes.names) keyframe_pairs = tuple(self.keyframes.track_pairs) sequential_pairs = tuple( - overlap_pair_plan( - keyframe_sequence, - track_options=self.track_options, - ) + pair + for stream in self.keyframes.streams + for pair in overlap_pair_plan(stream, track_options=self.track_options) ) + directed = {frozenset(pair): pair for pair in sequential_pairs} + sequential_pairs = tuple(directed[frozenset(pair)] for pair in combine_and_deduplicate_pairs(sequential_pairs)) validate_pair_name_plan(sequential_pairs) - track_metadata = self.keyframes.track_pairs_artifact.metadata salient_features_fingerprint = artifact_fingerprint(read_cache_metadata(paths.salient_features_path)) outputs = track_output_cache_metadata( self.track_options, self.highres_options, self.lowres_match_resolution, - self.extended_options, self.tracker_options, keyframe_sequence, keyframe_pairs, - track_metadata, + self.keyframes.admission_artifact.fingerprint, salient_features_fingerprint, ) - return _TrackCachePlan( - keyframe_sequence, - keyframe_pairs, - sequential_pairs, - track_metadata, - outputs, - ) + return _TrackCachePlan(keyframe_sequence, keyframe_pairs, sequential_pairs, outputs) def _outputs_complete(self, plan: _TrackCachePlan) -> bool: paths = self.paths @@ -254,31 +228,22 @@ def _outputs_complete(self, plan: _TrackCachePlan) -> bool: [names_to_pair(*pair) for pair in plan.keyframe_pairs], ) - def _extended_metadata(self, plan: _TrackCachePlan, sparse_features_metadata): + def _extended_metadata(self, sparse_features_metadata): return extended_matches_cache_metadata( - self.track_options, self.highres_options, self.lowres_match_resolution, self.extended_options, self.tracker_options, - plan.keyframe_sequence, - plan.sequential_pairs, - self.keyframes.track_pairs_artifact.metadata, sparse_features_metadata, ) - def _load(self, plan: _TrackCachePlan) -> ExtendedMatchCache: + def _load(self) -> ExtendedMatchCache: paths = self.paths extended_metadata = self._extended_metadata( - plan, read_cache_metadata(paths.sparse_features_path), ) - extended_cache = ExtendedMatchCache( - paths, - extended_metadata, - force_recompute=self.force_recompute, - ) - extended_cache.prepare() + extended_cache = ExtendedMatchCache(paths, force_recompute=self.force_recompute) + extended_cache.prepare(extended_metadata) return extended_cache def _publish(self, plan, staging_path): @@ -293,16 +258,14 @@ def _publish(self, plan, staging_path): plan.outputs.sparse_matches, [names_to_pair(*pair) for pair in plan.keyframe_pairs], ) - extended_cache = ExtendedMatchCache( - paths, - self._extended_metadata(plan, read_cache_metadata(paths.sparse_features_path)), - force_recompute=self.force_recompute, + extended_cache = ExtendedMatchCache(paths, force_recompute=self.force_recompute) + extended_cache.publish_staging( + staging_path, self._extended_metadata(read_cache_metadata(paths.sparse_features_path)) ) - extended_cache.publish_staging(staging_path) return extended_cache - def admit_and_build_tracks(self, candidates, keyframe_options, finalize_keyframes): - """Admit candidate keyframes while building their sparse tracks.""" + def admit_and_build_tracks(self, candidates, keyframe_options, finalize_keyframes, *, complete_salients): + """Admit synchronized frames, then rebuild every camera's sparse tracks.""" from vidmap.utils.profiling import log_memory, record_timing, sync_time paths = self.paths @@ -313,53 +276,50 @@ def admit_and_build_tracks(self, candidates, keyframe_options, finalize_keyframe prefix=".extended-matches-", ) as staging_dir: staging_path = Path(staging_dir) / paths.extended_matches_path.name - started = sync_time() - accepted_ids = StreamingTrackPropagator( + propagator = partial( + StreamingTrackPropagator, conf=self.track_options, scene_parser=self.scene_parser, - paths=paths, + paths=self.paths, tracker_model=self.tracker.get(), - keyframe_sequence=candidates.names, - candidate_indices=candidates.indices, - forced_candidate_indices=candidates.forced_indices, - calibrations=candidates.calibrations, - keyframe_options=keyframe_options, conf_highres=self.highres_options, lowres_match_resolution=self.lowres_match_resolution, extended_matches_path=staging_path, lc_match_thresh=self.extended_options.lc_match_thresh, device=self.device, + ) + started = sync_time() + accepted_ids = propagator( + keyframe_sequence=candidates.names, + candidate_indices=candidates.indices, + forced_candidate_indices=candidates.forced_indices, + calibrations=candidates.calibrations, + keyframe_options=keyframe_options, ).run() - record_timing("track_propagation", sync_time() - started) - log_memory("track_propagation") - + propagation_time = sync_time() - started self.keyframes = finalize_keyframes(accepted_ids) + complete_salients(self.keyframes) + for camera_index, stream in enumerate(self.keyframes.streams[1:], start=1): + logger.info( + "Tracking camera stream %d/%d: %d admitted frames", + camera_index + 1, + len(self.keyframes.streams), + len(stream), + ) + started = sync_time() + propagator(keyframe_sequence=stream).run() + propagation_time += sync_time() - started + record_timing("track_propagation", propagation_time) + log_memory("track_propagation") plan = self._cache_plan() extended_cache = self._publish(plan, staging_path) - - self._repair_sequential_matches(plan, extended_cache, loaded=False) return self.keyframes, self._result(plan, extended_cache) - def _repair_sequential_matches(self, plan: _TrackCachePlan, extended_cache: ExtendedMatchCache, *, loaded): - present_count, _ = extended_cache.repair( - plan.sequential_pairs, - label="sequential", - tracker=self.tracker, - scene_parser=self.scene_parser, - image_options=self.highres_options, - lowres_match_resolution=self.lowres_match_resolution, - match_threshold=self.extended_options.lc_match_thresh, - device=self.device, - ) - if loaded: - logger.info("Loaded %d cached track-overlap pairs", present_count) - def _result(self, plan: _TrackCachePlan, extended_cache: ExtendedMatchCache) -> SparseTrackResult: sparse_match_items = tuple(names_to_pair(*pair) for pair in plan.keyframe_pairs) result = SparseTrackResult( track_pairs=plan.keyframe_pairs, sequential_pairs=plan.sequential_pairs, - track_pairs_artifact=certify_complete_artifact(self.paths.track_pairs_path, plan.track_pairs), sparse_features=certify_incremental_artifact( self.paths.sparse_features_path, plan.outputs.sparse_features, @@ -387,6 +347,5 @@ def load_complete(self) -> SparseTrackResult | None: plan = self._cache_plan() if not self._outputs_complete(plan): return None - extended_cache = self._load(plan) - self._repair_sequential_matches(plan, extended_cache, loaded=True) + extended_cache = self._load() return self._result(plan, extended_cache) diff --git a/vidmap/frontend/tracking/transitive.py b/vidmap/frontend/tracking/transitive.py index b4de0aa..6bbfb51 100644 --- a/vidmap/frontend/tracking/transitive.py +++ b/vidmap/frontend/tracking/transitive.py @@ -1,80 +1,89 @@ """Build tracking-owned transitive correspondences from sparse tracks.""" import logging -from collections import defaultdict from collections.abc import Mapping from dataclasses import dataclass as result_dataclass from pathlib import Path from typing import Any +import h5py import numpy as np import pycolmap -from pycolmap import Database, DatabaseCache -from vidmap.datasets.base import DatasetParser +from vidmap.datasets.capture import PreparedCapture from vidmap.frontend.colmap_database import build_colmap_database from vidmap.frontend.correspondences import ImagePair, immutable_array_mapping -from vidmap.frontend.initial_reconstruction import build_initial_reconstruction from vidmap.frontend.keyframes.processing import KeyframePlan from vidmap.frontend.paths import FrontendPaths from vidmap.frontend.tracking.sparse_tracks import SparseTrackResult from vidmap.repro.frontend import write_sequence_artifact, write_sqlite_summary_artifact, write_tcorr_artifact +from vidmap.utils.io import get_matches_from_h5 logger = logging.getLogger(__name__) _TRANSITIVE_CORRESPONDENCE_DEPTH = 10_000_000 -def build_transitive_correspondences(graph, names_to_ids, sequence, transitivity_depth, database_path): - """Build transitive rows with the established set-deduplicated byte order.""" - correspondences = defaultdict(list) - with pycolmap.Database.open(str(database_path)) as database: - for name in sequence: +def build_correspondence_graph(source_reconstruction, keyframes, pairs, paths): + """Build COLMAP's graph directly, preserving SQLite's canonical pair traversal.""" + names_to_ids = {image.name: image_id for image_id, image in source_reconstruction.images.items()} + graph = pycolmap.CorrespondenceGraph() + counts = {} + with h5py.File(paths.sparse_features_path, "r", libver="latest") as features: + for name in keyframes: image_id = names_to_ids[name] - if not graph.exists_image(image_id): + counts[image_id] = features[name]["keypoints"].shape[0] + graph.add_image(image_id, counts[image_id]) + ordered_pairs = {} + for name0, name1 in pairs: + pair_id = pycolmap.image_pair_to_pair_id(names_to_ids[name0], names_to_ids[name1]) + if pair_id not in ordered_pairs: + ordered_pairs[pair_id] = (name0, name1) + with h5py.File(paths.sparse_matches_path, "r", libver="latest") as matches_file: + for pair_id, (name0, name1) in sorted(ordered_pairs.items()): + matches, _ = get_matches_from_h5(matches_file, name0, name1) + if not len(matches): continue - for point2d_id in range(database.read_keypoints(image_id).shape[0]): - # This is pycolmap's public method name, not VidMap frontend - # terminology, so retain the upstream API verb. - for match in graph.extract_transitive_correspondences(image_id, point2d_id, transitivity_depth): - if (match.image_id, image_id) in correspondences: - continue - correspondences[image_id, match.image_id].append((point2d_id, match.point2D_idx)) - return {pair: np.array(list(set(matches))) for pair, matches in correspondences.items() if matches} + if names_to_ids[name0] > names_to_ids[name1]: + matches = matches[:, ::-1] + graph.add_two_view_geometry( + *pycolmap.pair_id_to_image_pair(pair_id), + pycolmap.TwoViewGeometry( + config=pycolmap.TwoViewGeometryConfiguration.CALIBRATED, + inlier_matches=np.asarray(matches, dtype=np.uint32), + ), + ) + graph.finalize() + return graph, names_to_ids, counts + + +def build_transitive_correspondences(graph, names_to_ids, sequence, transitivity_depth, counts): + """Build ordered native rows and translate image IDs to frontend names.""" + from vidmap_native import _core + + ids = [names_to_ids[name] for name in sequence] + rows = _core.build_transitive_correspondences(graph, ids, [counts[image] for image in ids], transitivity_depth) + names = {image_id: name for name, image_id in names_to_ids.items()} + return {(names[a], names[b]): matches for (a, b), matches in rows.items()} @result_dataclass(frozen=True) class TransitiveCorrespondenceResult: """Transitive correspondence graph built from certified sparse tracks.""" - sequence: tuple[str, ...] tcorr: Mapping[ImagePair, Any] def __post_init__(self): - object.__setattr__(self, "sequence", tuple(self.sequence)) object.__setattr__(self, "tcorr", immutable_array_mapping(self.tcorr)) -def load_correspondence_graph(database_path, vidmap_rec): - """Load the COLMAP correspondence graph and populate reconstruction keypoints.""" - database = Database.open(str(database_path)) - cache = DatabaseCache.create(database, pycolmap.DatabaseCacheOptions()) - graph = cache.correspondence_graph - for image in database.read_all_images(): - keypoints = database.read_keypoints(image.image_id) - vidmap_rec.images[image.image_id].points2D = [ - pycolmap.Point2D(xy=keypoint) for keypoint in keypoints.astype(np.float16) - ] - return graph - - class TransitiveCorrespondenceBuilder: - """Build the transitive database and its immutable correspondence result.""" + """Build the native graph and its immutable correspondence result.""" def __init__( self, *, - scene_parser: DatasetParser, + scene_parser: PreparedCapture, paths: FrontendPaths, repro_dir: Path | None, keyframes: KeyframePlan, @@ -101,44 +110,30 @@ def build(self) -> TransitiveCorrespondenceResult: if missing_names: raise ValueError(f"Image {missing_names[0]} not found in scene_parser.rec.images") - database_path = self.paths.database_transitive_path - database_path.parent.mkdir(parents=True, exist_ok=True) - build_colmap_database( - database_path, - source_reconstruction, - keyframes, - self.paths.sparse_features_path, - pairs, - camera_policy="transitive", - prior_focal_length=True, - sparse_matches_path=self.paths.sparse_matches_path, - seed_two_view_geometry=True, + correspondence_graph, names_to_ids, counts = build_correspondence_graph( + source_reconstruction, keyframes, pairs, self.paths ) - - reconstruction = build_initial_reconstruction( - scene_parser=self.scene_parser, - reference_image_names=list(keyframe_names), - ) - correspondence_graph = load_correspondence_graph(database_path, vidmap_rec=reconstruction) sequence = keyframe_names - names_to_ids = {image.name: image_id for image_id, image in reconstruction.images.items()} tcorr = build_transitive_correspondences( - correspondence_graph, - names_to_ids, - sequence, - _TRANSITIVE_CORRESPONDENCE_DEPTH, - database_path, + correspondence_graph, names_to_ids, sequence, _TRANSITIVE_CORRESPONDENCE_DEPTH, counts ) - tcorr = { - ( - reconstruction.images[pair[0]].name, - reconstruction.images[pair[1]].name, - ): matches - for pair, matches in tcorr.items() - } logger.info("Built %d transitive correspondence pairs", len(tcorr)) if self.repro_dir is not None: + database_path = self.paths.database_transitive_path + database_path.parent.mkdir(parents=True, exist_ok=True) + build_colmap_database( + database_path, + source_reconstruction, + keyframes, + self.paths.sparse_features_path, + pairs, + diagnostic_transitive=True, + prior_focal_length=True, + sparse_matches_path=self.paths.sparse_matches_path, + seed_two_view_geometry=True, + ) + write_sequence_artifact( self.repro_dir / "stage1_transitive_sequence_order.json", sequence, @@ -150,4 +145,4 @@ def build(self) -> TransitiveCorrespondenceResult: database_path, label="database_transitive", ) - return TransitiveCorrespondenceResult(tuple(sequence), tcorr) + return TransitiveCorrespondenceResult(tcorr) diff --git a/vidmap/mapper/focal_prior.py b/vidmap/mapper/focal_prior.py index 5953ff9..f776346 100644 --- a/vidmap/mapper/focal_prior.py +++ b/vidmap/mapper/focal_prior.py @@ -19,22 +19,23 @@ def native_focal_priors(prior, *, camera_ids, loss, scale=1.0, weight=1.0): return records -def load_focal_prior(path, state, *, log_focal_stddev=None, shared=False): - images = state.reconstruction.images +def load_focal_prior(path, state, evidence, *, log_focal_stddev=None): cameras = state.reconstruction.cameras - if shared and len(cameras) != 1: - raise ValueError("Selected shared calibration requires one camera") - associations = ( - [("batch_calibration", next(iter(cameras)))] - if shared - else [(images[i].name, images[i].camera_id) for i in state.image_order] - ) - if not associations: - raise ValueError("Calibration results require camera associations") + images = {image.name: image for image in state.reconstruction.images.values()} + shared = evidence.inference == "selected_batch" + ordered = evidence.observations + if not shared: + by_name = {observation.image_name: observation for observation in ordered} + ordered = tuple(by_name.pop(state.reconstruction.images[i].name) for i in state.image_order) + if by_name or len(ordered) != len(evidence.observations): + raise ValueError("Per-view calibration observations disagree with database images") observations = {} with h5py.File(path, "r") as hfile: - for name, camera_id in associations: - result = hfile[name] + for observation in ordered: + camera_id = observation.camera_id + if observation.image_name is not None and images[observation.image_name].camera_id != camera_id: + raise ValueError("Calibration observation disagrees with database camera membership") + result = hfile[observation.key] camera = cameras[camera_id] if tuple(result["image_size"][:]) != (camera.width, camera.height): raise ValueError("Calibration dimensions disagree with database") diff --git a/vidmap/mapper/inputs/__init__.py b/vidmap/mapper/inputs/__init__.py index a7a0760..78f4369 100644 --- a/vidmap/mapper/inputs/__init__.py +++ b/vidmap/mapper/inputs/__init__.py @@ -14,7 +14,6 @@ REQUIRED_NAMES, REQUIRED_PAYLOAD_NAMES, SQLITE_SIDECAR_SUFFIXES, - TRACK_PAIRS_NAME, VGC_FILTERED_PAIRS_NAME, FileProvenance, MapperInputs, @@ -40,7 +39,6 @@ "REQUIRED_NAMES", "REQUIRED_PAYLOAD_NAMES", "SQLITE_SIDECAR_SUFFIXES", - "TRACK_PAIRS_NAME", "VGC_FILTERED_PAIRS_NAME", "FileProvenance", "MapperInputs", diff --git a/vidmap/mapper/inputs/database.py b/vidmap/mapper/inputs/database.py index 6f84c96..b223f76 100644 --- a/vidmap/mapper/inputs/database.py +++ b/vidmap/mapper/inputs/database.py @@ -40,8 +40,17 @@ def load_finalized_database( sidecars = native.MappingSidecars() pose_graph = pycolmap.PoseGraph() + rigs = database.read_all_rigs() + frames = database.read_all_frames() + if not rigs or not frames: + raise ValueError("Finalized database requires native rig/frame metadata") for cam in database.read_all_cameras(): - rec.add_camera_with_trivial_rig(cam) + rec.add_camera(cam) + for rig in rigs: + rec.add_rig(rig) + for frame in frames: + frame.rig_from_world = pycolmap.Rigid3d(translation=np.full(3, np.nan)) + rec.add_frame(frame) images_colmap = database.read_all_images() logger.info("Loading %d images from the finalized database", len(images_colmap)) @@ -55,7 +64,8 @@ def load_finalized_database( image_id=image_id, keypoints=features, ) - rec.add_image_with_trivial_frame(gimg, pycolmap.Rigid3d(translation=np.full(3, np.nan))) + gimg.frame_id = img.frame_id + rec.add_image(gimg) sidecars.add_image(image_id, native.ImageData()) pair_ids, matches = database.read_all_matches() total_pairs = len(pair_ids) @@ -78,7 +88,7 @@ def load_finalized_database( rec.image(img1_id) rec.image(img2_id) data = native.PairData() - data.geometry = pycolmap.TwoViewGeometry(config=cfg, F=two_view.F, H=two_view.H) + data.geometry = pycolmap.TwoViewGeometry(config=cfg, F=two_view.F, H=two_view.H, E=two_view.E) data.all_matches = np.asarray(feat_matches, dtype=np.uint32).reshape((-1, 2)) sidecars.add_pair(int(pair_id), data) edge = pycolmap.PoseGraphEdge() diff --git a/vidmap/mapper/inputs/loader.py b/vidmap/mapper/inputs/loader.py index 62a02eb..664e7ba 100644 --- a/vidmap/mapper/inputs/loader.py +++ b/vidmap/mapper/inputs/loader.py @@ -13,6 +13,7 @@ from vidmap.mapper.inputs import MapperInputs from vidmap.mapper.inputs.database import copy_finalized_database, load_finalized_database +from vidmap.mapper.inputs.timeline import MappingTimeline, mapping_timeline from vidmap.mapper.native.extension import native from vidmap.mapper.native.state import SolveState from vidmap.mapper.options.mapper import SetupOptions @@ -20,7 +21,6 @@ from vidmap.mapper.replay.evidence.canonical import snapshot_images from vidmap.mapper.replay.evidence.stages import database_file_summary, database_to_native_summary from vidmap.utils.image_sampling import sample_at_keypoints -from vidmap.utils.io import ordered_pair_images from vidmap.utils.loop_closure_masks import read_loop_closure_masks logger = logging.getLogger(__name__) @@ -32,7 +32,7 @@ class MappingStageInputs: solve_state: SolveState consecutive_pair_ids: list[int] - sequence_id_to_index: dict[int, int] + timeline: MappingTimeline vgc_exclusion_ids: set[int] @@ -120,7 +120,6 @@ def attach_depth_inputs(self, state: SolveState, depths) -> None: def load(self, *, on_database_loaded: Callable[[SolveState], None] | None = None) -> MappingStageInputs: self.inputs.validate() database_path = self.inputs.database_path - consecutive_pairs = self.inputs.read_track_pairs() lc_masks = read_loop_closure_masks(self.inputs.lc_masks_path) vgc_filtered_pairs = self.inputs.read_vgc_filtered_pairs() if not database_path.exists(): @@ -193,22 +192,11 @@ def load(self, *, on_database_loaded: Callable[[SolveState], None] | None = None len(vgc_exclusion_ids), ) - # The temporal track-pair file must describe exactly one adjacent chain - # over the finalized database images. - images = state.reconstruction.images - sequence_names = ordered_pair_images(consecutive_pairs) - if consecutive_pairs != list(zip(sequence_names, sequence_names[1:])): - raise ValueError("Mapper track pairs must form one ordered adjacent image chain") - name_to_id = {image.name: image_id for image_id, image in images.items()} - if set(sequence_names) != set(name_to_id): - raise ValueError("Mapper track-pair sequence does not match finalized database images") - sequence_ids = [name_to_id[name] for name in sequence_names] - sequence_id_to_index = {sid: idx for idx, sid in enumerate(sequence_ids)} + timeline = mapping_timeline(self.inputs.frame_sequence(), state.reconstruction) consecutive_pair_ids = [] - for pair in consecutive_pairs: - pair_id = pair_name_to_pid.get(frozenset(pair)) - if pair_id is not None: + for pair_id in timeline.consecutive_pair_ids: + if pair_id in state.pose_graph.edges: consecutive_pair_ids.append(pair_id) # Finalize native pair classifications only @@ -218,6 +206,6 @@ def load(self, *, on_database_loaded: Callable[[SolveState], None] | None = None return MappingStageInputs( solve_state=state, consecutive_pair_ids=consecutive_pair_ids, - sequence_id_to_index=sequence_id_to_index, + timeline=timeline, vgc_exclusion_ids=vgc_exclusion_ids, ) diff --git a/vidmap/mapper/inputs/snapshot.py b/vidmap/mapper/inputs/snapshot.py index ee4cce3..b51709c 100644 --- a/vidmap/mapper/inputs/snapshot.py +++ b/vidmap/mapper/inputs/snapshot.py @@ -10,24 +10,23 @@ from functools import lru_cache from pathlib import Path -import h5py - +from vidmap.calibration import CalibrationEvidence from vidmap.configuration.names import config_name_to_output_slug from vidmap.depth_artifacts import FULL_DEPTH_MAPS_NAME +from vidmap.mapper.inputs.timeline import validate_frame_sequence DATABASE_NAME = "database_complete.db" -TRACK_PAIRS_NAME = "track_pairs.h5" DEPTH_MAPS_NAME = "depth_maps.h5" LC_MASKS_NAME = "lc_masks.json" GEOCALIB_PER_IMAGE_NAME = "geocalib_per_image.h5" GEOCALIB_BATCH_NAME = "geocalib_batch.h5" VGC_FILTERED_PAIRS_NAME = "vgc_filtered_pairs.json" MANIFEST_NAME = "manifest.json" -MANIFEST_SCHEMA_VERSION = 10 -_FRONTEND_IDENTITY_SCHEMA_VERSION = 4 +MANIFEST_SCHEMA_VERSION = 14 +_FRONTEND_IDENTITY_SCHEMA_VERSION = 6 SQLITE_SIDECAR_SUFFIXES = ("-journal", "-shm", "-wal") -REQUIRED_PAYLOAD_NAMES = frozenset({DATABASE_NAME, TRACK_PAIRS_NAME, DEPTH_MAPS_NAME, LC_MASKS_NAME}) +REQUIRED_PAYLOAD_NAMES = frozenset({DATABASE_NAME, DEPTH_MAPS_NAME, LC_MASKS_NAME}) REQUIRED_NAMES = REQUIRED_PAYLOAD_NAMES | {MANIFEST_NAME} OPTIONAL_NAMES = frozenset({GEOCALIB_PER_IMAGE_NAME, GEOCALIB_BATCH_NAME, VGC_FILTERED_PAIRS_NAME}) CANONICAL_NAMES = REQUIRED_NAMES | OPTIONAL_NAMES @@ -43,10 +42,9 @@ "mode", "testset_id", "reference_image_ids", - "boundary_options", + "source_fingerprint", } ) -BOUNDARY_OPTION_KEYS = frozenset({"estimator", "inference", "initialization"}) def calibration_artifact_name(estimator: str, inference: str) -> str | None: @@ -104,17 +102,17 @@ def mapper_inputs_manifest( directory: Path, *, frontend_identity: Mapping[str, object], + frame_sequence: dict, + calibration: CalibrationEvidence | None = None, ) -> dict: """Build deterministic provenance for canonical mapper-input payloads.""" - directory = Path(directory) - artifacts = {} - for name in sorted(CANONICAL_NAMES - {MANIFEST_NAME}): - path = directory / name - if not path.is_file(): - continue - proof = FileProvenance.from_path(path) - artifacts[path.name] = {"size": proof.size, "sha256": proof.sha256} - return _tagged_manifest(artifacts, frontend_identity) + directory = resolve_mapper_inputs_directory(directory) + names = {name for name in CANONICAL_NAMES - {MANIFEST_NAME} if (directory / name).is_file()} + artifacts = { + name: dict(proof) + for name, proof in _verified_artifacts(directory, _artifact_signature(directory, names)).items() + } + return _tagged_manifest(artifacts, frontend_identity, frame_sequence, calibration) def _artifact_signature( @@ -144,13 +142,19 @@ def _verified_artifacts( def _tagged_manifest( artifacts: Mapping[str, object], frontend_identity: Mapping[str, object], + frame_sequence: dict, + calibration: CalibrationEvidence | None = None, ) -> dict[str, object]: """Build the current tagged mapper-input manifest.""" - return { + validate_frame_sequence(frame_sequence) + manifest = { "schema_version": MANIFEST_SCHEMA_VERSION, "frontend_identity": dict(frontend_identity), "artifacts": dict(artifacts), + "frame_sequence": frame_sequence, + "calibration": None if calibration is None else calibration.as_dict(), } + return manifest def _tagged_identity(manifest: Mapping[str, object], *, manifest_path: Path) -> dict[str, object]: @@ -158,6 +162,10 @@ def _tagged_identity(manifest: Mapping[str, object], *, manifest_path: Path) -> schema_version = manifest.get("schema_version") if schema_version != MANIFEST_SCHEMA_VERSION: raise ValueError(f"Mapper inputs require manifest schema {MANIFEST_SCHEMA_VERSION}: {manifest_path}") + fields = {"schema_version", "frontend_identity", "artifacts", "frame_sequence", "calibration"} + if set(manifest) != fields: + raise ValueError(f"Invalid mapper-input manifest fields: {manifest_path}") + validate_frame_sequence(manifest["frame_sequence"]) identity = manifest.get("frontend_identity") _validate_frontend_identity(identity, manifest_path=manifest_path) return identity @@ -167,8 +175,15 @@ def write_mapper_inputs_manifest( directory: Path, *, frontend_identity: Mapping[str, object], + frame_sequence: dict, + calibration: CalibrationEvidence | None = None, ) -> None: - payload = mapper_inputs_manifest(directory, frontend_identity=frontend_identity) + payload = mapper_inputs_manifest( + directory, + frontend_identity=frontend_identity, + frame_sequence=frame_sequence, + calibration=calibration, + ) (Path(directory) / MANIFEST_NAME).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") @@ -221,25 +236,15 @@ def _validate_frontend_identity(identity: object, *, manifest_path: Path) -> Non or config_name_to_output_slug(config_name) != tag ): raise ValueError(f"Mapper-input frontend tag does not match config_name: {manifest_path}") - fingerprint = identity["config_fingerprint"] - if not isinstance(fingerprint, str) or re.fullmatch(r"[0-9a-f]{64}", fingerprint) is None: - raise ValueError(f"Invalid mapper-input frontend config_fingerprint: {manifest_path}") + for key in ("config_fingerprint", "source_fingerprint"): + fingerprint = identity[key] + if not isinstance(fingerprint, str) or re.fullmatch(r"[0-9a-f]{64}", fingerprint) is None: + raise ValueError(f"Invalid mapper-input frontend {key}: {manifest_path}") reference_image_ids = identity["reference_image_ids"] if not isinstance(reference_image_ids, list) or any( not isinstance(image_id, int) or isinstance(image_id, bool) for image_id in reference_image_ids ): raise ValueError(f"Invalid mapper-input frontend reference_image_ids: {manifest_path}") - options = identity["boundary_options"] - if ( - not isinstance(options, dict) - or set(options) != BOUNDARY_OPTION_KEYS - or options["estimator"] not in {"da3", "geocalib", "none"} - or options["inference"] not in {"per_view", "selected_batch"} - or options["initialization"] not in {"predicted", "supplied"} - or (options["inference"] == "selected_batch" and options["estimator"] != "geocalib") - or (options["estimator"] == "none" and options["initialization"] != "supplied") - ): - raise ValueError(f"Invalid mapper-input frontend boundary_options: {manifest_path}") def resolve_mapper_inputs_directory(directory: Path) -> Path: @@ -287,6 +292,7 @@ def validate(self) -> None: if not path.is_file(): raise ValueError(f"Mapper input must be a regular file: {path}") + require_finalized_sqlite(self.database_path) if VGC_FILTERED_PAIRS_NAME in names: self.read_vgc_filtered_pairs() @@ -307,8 +313,10 @@ def validate(self) -> None: if schema_version != MANIFEST_SCHEMA_VERSION: raise ValueError(f"Unsupported mapper-input manifest schema in {manifest_path}: {schema_version!r}") frontend_identity = _tagged_identity(manifest, manifest_path=manifest_path) - calibration = frontend_identity["boundary_options"] - selected_artifact = calibration_artifact_name(calibration["estimator"], calibration["inference"]) + evidence = None if manifest["calibration"] is None else CalibrationEvidence.from_dict(manifest["calibration"]) + selected_artifact = ( + None if evidence is None else calibration_artifact_name(evidence.estimator, evidence.inference) + ) expected_geocalib = {selected_artifact} & {GEOCALIB_PER_IMAGE_NAME, GEOCALIB_BATCH_NAME} if names & {GEOCALIB_PER_IMAGE_NAME, GEOCALIB_BATCH_NAME} != expected_geocalib: raise ValueError("GeoCalib artifacts disagree with the extraction's predictor settings") @@ -326,6 +334,8 @@ def validate(self) -> None: _artifact_signature(self.directory, canonical_names - {MANIFEST_NAME}), ), frontend_identity, + manifest["frame_sequence"], + evidence, ) if manifest != expected_manifest: raise ValueError(f"Mapper-input manifest does not match written payloads: {manifest_path}") @@ -339,21 +349,18 @@ def frontend_identity(self) -> dict[str, object]: raise ValueError(f"Invalid mapper-input manifest: {manifest_path}") from error return _tagged_identity(manifest, manifest_path=manifest_path) - def read_track_pairs(self) -> list[tuple[str, str]]: - with h5py.File(self.track_pairs_path, "r") as hfile: - if set(hfile) != {"data"}: - raise ValueError(f"Mapper track-pair file has an invalid dataset set: {self.track_pairs_path}") - data = hfile["data"][:] - if data.ndim != 2 or data.shape[1] != 2: - raise ValueError(f"Mapper track pairs must have shape (N, 2): {self.track_pairs_path}") - pairs = [] - for first, second in data: - if isinstance(first, bytes): - first = first.decode("utf-8") - if isinstance(second, bytes): - second = second.decode("utf-8") - pairs.append((str(first), str(second))) - return pairs + def calibration(self) -> CalibrationEvidence | None: + """Read original prediction associations without inferring camera membership.""" + manifest = json.loads((self.directory / MANIFEST_NAME).read_text()) + _tagged_identity(manifest, manifest_path=self.directory / MANIFEST_NAME) + value = manifest["calibration"] + return None if value is None else CalibrationEvidence.from_dict(value) + + def frame_sequence(self) -> dict: + """Read the frontend's explicit order over finalized native frames.""" + manifest = json.loads((self.directory / MANIFEST_NAME).read_text()) + _tagged_identity(manifest, manifest_path=self.directory / MANIFEST_NAME) + return manifest["frame_sequence"] @property def full_depth_maps_path(self) -> Path | None: @@ -385,10 +392,6 @@ def read_vgc_filtered_pairs(self) -> set[tuple[str, str]] | None: def database_path(self) -> Path: return self.directory / DATABASE_NAME - @property - def track_pairs_path(self) -> Path: - return self.directory / TRACK_PAIRS_NAME - @property def depth_maps_path(self) -> Path: return self.directory / DEPTH_MAPS_NAME diff --git a/vidmap/mapper/inputs/timeline.py b/vidmap/mapper/inputs/timeline.py new file mode 100644 index 0000000..55a8654 --- /dev/null +++ b/vidmap/mapper/inputs/timeline.py @@ -0,0 +1,75 @@ +# Strict integer checks intentionally reject bool. +# ruff: noqa: E721 +"""One explicit temporal order over native COLMAP frames.""" + +from collections.abc import Mapping +from dataclasses import dataclass + + +@dataclass +class MappingTimeline: + """Validated frame order and its single image-index projection.""" + + frame_ids: tuple[int, ...] + timestamps_ns: tuple[int, ...] | None + image_indices: Mapping[int, int] + consecutive_pair_ids: tuple[int, ...] + + +def validate_frame_sequence(sequence): + if not isinstance(sequence, dict) or set(sequence) != {"frame_ids", "timestamps_ns"}: + raise ValueError("Frame sequence requires frame_ids and timestamps_ns") + ids, times = sequence["frame_ids"], sequence["timestamps_ns"] + if ( + not isinstance(ids, list) + or not ids + or any(type(fid) is not int or not 0 <= fid < 2**32 - 1 for fid in ids) + or len(set(ids)) != len(ids) + ): + raise ValueError("Frame sequence requires unique native frame IDs") + if times is not None and ( + not isinstance(times, list) + or len(times) != len(ids) + or any(type(time) is not int for time in times) + or any(b <= a for a, b in zip(times, times[1:])) + ): + raise ValueError("Frame sequence timestamps must increase and cover every frame") + + +def mapping_timeline(sequence, reconstruction): + """Validate temporal chains and project frame order to images for every input.""" + import pycolmap + + validate_frame_sequence(sequence) + ids, times = sequence["frame_ids"], sequence["timestamps_ns"] + if set(ids) != set(reconstruction.frames) or not reconstruction.is_valid(): + raise ValueError("Frame sequence must cover exactly the valid native reconstruction") + previous = None + consecutive, indices = [], {} + for index, fid in enumerate(ids): + frame = reconstruction.frame(fid) + members = sorted(frame.image_ids, key=lambda data: data.sensor_id.id) + sensors = set(reconstruction.rig(frame.rig_id).sensor_ids()) + if not members or {data.sensor_id for data in members} != sensors or len(members) != len(sensors): + raise ValueError("Frame sequence requires complete camera frames") + images = [reconstruction.image(data.id) for data in members] + if any(image.frame_id != fid for image in images): + raise ValueError("Frame sequence and native image membership disagree") + if previous is not None: + # A singleton stream may have independent calibration at each time. + if len(images) != len(previous) or ( + len(images) > 1 and [image.camera_id for image in images] != [image.camera_id for image in previous] + ): + raise ValueError("Frame sequence must preserve synchronized camera streams") + consecutive.extend( + pycolmap.image_pair_to_pair_id(a.image_id, b.image_id) for a, b in zip(previous, images, strict=True) + ) + for image in images: + indices[image.image_id] = index + previous = images + return MappingTimeline( + tuple(ids), + None if times is None else tuple(times), + indices, + tuple(consecutive), + ) diff --git a/vidmap/mapper/mapper.py b/vidmap/mapper/mapper.py index 63d4313..35de260 100644 --- a/vidmap/mapper/mapper.py +++ b/vidmap/mapper/mapper.py @@ -67,8 +67,8 @@ def run( overwrite_outputs: bool = False, on_inputs_validated: Callable[[], None] | None = None, ): - boundary = self.mapper_inputs.frontend_identity()["boundary_options"] - need_factors = self._validate_calibration_request(boundary) + evidence = self.mapper_inputs.calibration() + need_factors = self._validate_calibration_request(evidence) if save_playback_trace: PlaybackTraceRecorder.preflight(self.sfm_outputs_dir, replace=overwrite_outputs) playback_options = PlaybackTraceOptions( @@ -91,7 +91,7 @@ def run( def admit_calibration(state): nonlocal prior if need_factors: - prior = self._prepare_calibration(state, boundary) + prior = self._prepare_calibration(state, evidence) mapping_stage_inputs = loader.load(on_database_loaded=admit_calibration) @@ -125,10 +125,10 @@ def admit_calibration(state): remove_database_sidecars(working_database) working_database.unlink(missing_ok=True) - def _validate_calibration_request(self, boundary) -> bool: + def _validate_calibration_request(self, evidence) -> bool: """Check configuration before loading or publishing and resolve whether factors are needed.""" calibration = self.conf.calibration - predictor = boundary["estimator"] + predictor = "none" if evidence is None else evidence.estimator if calibration.vgc_focal_prior and (predictor != "da3" or not calibration.vgc_enabled): raise ValueError("VGC focal prior requires enabled DA3 VGC") need_factors = calibration.vgc_focal_prior or ( @@ -138,17 +138,18 @@ def _validate_calibration_request(self, boundary) -> bool: raise ValueError("Focal regularization requires original predictor factors") return need_factors - def _prepare_calibration(self, solve_state, boundary): + def _prepare_calibration(self, solve_state, evidence): """Construct original-target factors from the validated mapper inputs.""" calibration = self.conf.calibration - predictor = boundary["estimator"] + predictor = evidence.estimator cameras = solve_state.reconstruction.cameras - if any(c.model.name not in {"PINHOLE", "SIMPLE_PINHOLE"} for c in cameras.values()): - raise ValueError("Calibration requires pinhole cameras") - shared = boundary["inference"] == "selected_batch" - path = self.mapper_inputs.directory / calibration_artifact_name(predictor, boundary["inference"]) + models = {"PINHOLE", "SIMPLE_PINHOLE"} + camera_ids = {observation.camera_id for observation in evidence.observations} + if any(cameras[camera_id].model.name not in models for camera_id in camera_ids): + raise ValueError("Calibration prediction model disagrees with database cameras") + path = self.mapper_inputs.directory / calibration_artifact_name(predictor, evidence.inference) log_focal_stddev = calibration.da3_log_focal_stddev if predictor == "da3" else None - return load_focal_prior(path, solve_state, log_focal_stddev=log_focal_stddev, shared=shared) + return load_focal_prior(path, solve_state, evidence, log_focal_stddev=log_focal_stddev) def _solve(self, mapping_stage_inputs, replay, playback_trace, prior): solve_start_time = sync_time() @@ -197,7 +198,7 @@ def _solve(self, mapping_stage_inputs, replay, playback_trace, prior): rotation_averager = RotationAverager( solve_state=solve_state, options=self.conf.ra, - sequence_id_to_index=mapping_stage_inputs.sequence_id_to_index, + sequence_id_to_index=mapping_stage_inputs.timeline.image_indices, filtered_consecutive_pair_ids=relative_pose.filtered_consecutive_pairs, replay=replay, ) @@ -209,7 +210,7 @@ def _solve(self, mapping_stage_inputs, replay, playback_trace, prior): solve_state=solve_state, options=self.conf.depth_consistency, consecutive_pair_ids=mapping_stage_inputs.consecutive_pair_ids, - sequence_id_to_index=mapping_stage_inputs.sequence_id_to_index, + sequence_id_to_index=mapping_stage_inputs.timeline.image_indices, ) boundary_depth_outliers_marked = depth_consistency_filter.filter() @@ -229,7 +230,7 @@ def _solve(self, mapping_stage_inputs, replay, playback_trace, prior): global_positioner = GlobalPositioner( solve_state=solve_state, consecutive_pair_ids=mapping_stage_inputs.consecutive_pair_ids, - sequence_id_to_index=mapping_stage_inputs.sequence_id_to_index, + timeline=mapping_stage_inputs.timeline, inlier_thresholds=relative_pose.inlier_thresholds, boundary_depth_outliers_marked=boundary_depth_outliers_marked, options=self.conf.gp, diff --git a/vidmap/mapper/stages/bundle_adjustment/adjuster.py b/vidmap/mapper/stages/bundle_adjustment/adjuster.py index e926f98..2d29077 100644 --- a/vidmap/mapper/stages/bundle_adjustment/adjuster.py +++ b/vidmap/mapper/stages/bundle_adjustment/adjuster.py @@ -473,14 +473,14 @@ def solve_problem( depth_batches = [] depth_scales = [] - images_without_residuals = [] + observed_frames = set() small_angle_ids = self.small_triangulation_angle_ids(depth_options.risky_triangulation_angle_deg) for image_id in optimized_image_ids: image = self.reconstruction.images[image_id] point2D_indices = np.array(image.get_observation_point2D_idxs()) if point2D_indices.size == 0: - images_without_residuals.append(image_id) continue + observed_frames.add(image.frame_id) solve_image = self.solve_state.image_data(image_id) valid = np.asarray(solve_image.depth_validity, dtype=bool)[point2D_indices] depths = np.asarray(solve_image.depth_values)[point2D_indices] @@ -576,8 +576,10 @@ def solve_problem( return False self.log_depth_scales.update(result.log_depth_scales) if not policy.fix_all_poses: - for image_id in images_without_residuals: - self.reconstruction.deregister_frame(self.reconstruction.image(image_id).frame_id) + frame_order = dict.fromkeys(self.reconstruction.images[i].frame_id for i in optimized_image_ids) + for frame_id in frame_order: + if frame_id not in observed_frames: + self.reconstruction.deregister_frame(frame_id) logger.debug( "BA solved: residuals=%d, initial_cost=%.8g, final_cost=%.8g", result.diagnostics.num_residual_blocks, diff --git a/vidmap/mapper/stages/global_positioning/positioner.py b/vidmap/mapper/stages/global_positioning/positioner.py index 3daa7c9..0fa7150 100644 --- a/vidmap/mapper/stages/global_positioning/positioner.py +++ b/vidmap/mapper/stages/global_positioning/positioner.py @@ -13,6 +13,7 @@ from vidmap_native import global_positioning as gp_costs from vidmap.mapper.checkpoints import reconstruction_checkpoint_directory +from vidmap.mapper.inputs.timeline import MappingTimeline from vidmap.mapper.native.extension import native from vidmap.mapper.native.state import SolveState from vidmap.mapper.options.positioning import GPOptions @@ -68,60 +69,77 @@ def _build_temporal_acceleration_prior_specs( solve_state: SolveState, reconstruction, consecutive_pair_ids, - sequence_id_to_index, + timeline: MappingTimeline, coordinate: str = "timestamp", ) -> list: """Build smoothness priors across valid consecutive tracking edges.""" - ordered_image_ids = [image_id for image_id, _ in sorted(sequence_id_to_index.items(), key=lambda item: item[1])] - if coordinate == "timestamp": - timeline = _image_timestamps_seconds([reconstruction.image(image_id).name for image_id in ordered_image_ids]) - coordinates = dict(zip(ordered_image_ids, timeline, strict=True)) - elif coordinate == "keyframe_index": - coordinates = {image_id: index for index, image_id in enumerate(ordered_image_ids)} - else: + if coordinate not in ("timestamp", "keyframe_index"): raise ValueError(f"Unsupported temporal-acceleration coordinate: {coordinate!r}") - registered_image_ids = {image_id for image_id in ordered_image_ids if reconstruction.image(image_id).has_pose} + ordered_frame_ids = timeline.frame_ids + frame_indices = {fid: index for index, fid in enumerate(ordered_frame_ids)} + if coordinate == "keyframe_index": + coordinates = dict(frame_indices) + elif timeline.timestamps_ns is not None: + origin = timeline.timestamps_ns[0] + coordinates = { + frame: Decimal(time - origin) / Decimal(1_000_000_000) + for frame, time in zip(ordered_frame_ids, timeline.timestamps_ns, strict=True) + } + else: + if gp_costs.validate_fixed_rigs(reconstruction): + raise ValueError("Rig temporal acceleration requires recorded timestamps for timestamp coordinates") + names = [ + reconstruction.image(next(iter(reconstruction.frame(fid).image_ids)).id).name for fid in ordered_frame_ids + ] + coordinates = dict(zip(ordered_frame_ids, _image_timestamps_seconds(names), strict=True)) + registered_frame_ids = {frame_id for frame_id in ordered_frame_ids if reconstruction.frame(frame_id).has_pose()} valid_edges = set() for pair_id in consecutive_pair_ids: image_id1, image_id2 = pycolmap.pair_id_to_image_pair(pair_id) if not solve_state.pose_graph.has_edge(image_id1, image_id2) or not solve_state.pose_graph.is_valid(pair_id): continue - index1 = sequence_id_to_index[image_id1] - index2 = sequence_id_to_index[image_id2] + frame_id1 = reconstruction.image(image_id1).frame_id + frame_id2 = reconstruction.image(image_id2).frame_id + index1 = frame_indices[frame_id1] + index2 = frame_indices[frame_id2] if abs(index1 - index2) == 1: - valid_edges.add(frozenset((image_id1, image_id2))) + valid_edges.add(frozenset((frame_id1, frame_id2))) observation_counts = gp_costs.track_observation_counts(reconstruction, solve_state.sidecars) - for image_id in ordered_image_ids: - if image_id not in observation_counts: - observation_counts[image_id] = 0 + frame_counts = { + fid: sum( + observation_counts[data.id] + for data in reconstruction.frame(fid).image_ids + if data.id in observation_counts + ) + for fid in ordered_frame_ids + } priors = [] - for prev_image_id, image_id, next_image_id in zip( - ordered_image_ids, - ordered_image_ids[1:], - ordered_image_ids[2:], + for prev_frame_id, frame_id, next_frame_id in zip( + ordered_frame_ids, + ordered_frame_ids[1:], + ordered_frame_ids[2:], ): - triplet = (prev_image_id, image_id, next_image_id) - if any(candidate not in registered_image_ids for candidate in triplet): + triplet = (prev_frame_id, frame_id, next_frame_id) + if any(candidate not in registered_frame_ids for candidate in triplet): continue - if frozenset((prev_image_id, image_id)) not in valid_edges: + if frozenset((prev_frame_id, frame_id)) not in valid_edges: continue - if frozenset((image_id, next_image_id)) not in valid_edges: + if frozenset((frame_id, next_frame_id)) not in valid_edges: continue - mean_observation_count = sum(observation_counts[candidate] for candidate in triplet) / 3.0 + mean_observation_count = sum(frame_counts[candidate] for candidate in triplet) / 3.0 if mean_observation_count <= 0.0: continue - dt_prev = float(coordinates[image_id] - coordinates[prev_image_id]) - dt_next = float(coordinates[next_image_id] - coordinates[image_id]) + dt_prev = float(coordinates[frame_id] - coordinates[prev_frame_id]) + dt_next = float(coordinates[next_frame_id] - coordinates[frame_id]) if not np.isfinite(dt_prev) or not np.isfinite(dt_next) or dt_prev <= 0.0 or dt_next <= 0.0: raise ValueError( f"temporal acceleration produced invalid {coordinate} gaps for " - f"{solve_state.image(prev_image_id).name!r}, {reconstruction.image(image_id).name!r}, " - f"{solve_state.image(next_image_id).name!r}" + f"frames {prev_frame_id}, {frame_id}, {next_frame_id}" ) priors.append( TemporalAccelerationPrior( - prev_image_id, image_id, next_image_id, dt_prev, dt_next, float(np.sqrt(mean_observation_count)) + prev_frame_id, frame_id, next_frame_id, dt_prev, dt_next, float(np.sqrt(mean_observation_count)) ) ) return priors @@ -164,7 +182,7 @@ class GlobalPositioner: solve_state: SolveState consecutive_pair_ids: list[int] - sequence_id_to_index: dict[int, int] + timeline: MappingTimeline inlier_thresholds: InlierThresholdOptions boundary_depth_outliers_marked: bool options: GPOptions @@ -323,7 +341,7 @@ def position(self) -> None: solve_state=self.solve_state, reconstruction=working, consecutive_pair_ids=self.consecutive_pair_ids, - sequence_id_to_index=self.sequence_id_to_index, + timeline=self.timeline, coordinate=temporal.coordinate, ) logger.info( @@ -334,10 +352,7 @@ def position(self) -> None: max_depth = self.options.track_filter.depth_prior_outlier_max_depth depth_masks = _depth_prior_outlier_masks(self.solve_state, max_depth) - image_timeline = [ - int(image_id) for image_id, _ in sorted(self.sequence_id_to_index.items(), key=lambda item: item[1]) - ] - prepared = prepare_observations(working, self.solve_state.sidecars, self.options, image_timeline) + prepared = prepare_observations(working, self.solve_state.sidecars, self.options, self.timeline.frame_ids) # GP1: initialize and solve, with optional warmup. logger.info("Running first global positioning ...") result = self.run_pass("gp1", working, prepared, replay_images, depth_masks, temporal_prior_specs) diff --git a/vidmap/mapper/stages/global_positioning/problem.py b/vidmap/mapper/stages/global_positioning/problem.py index 0c6c166..fc48716 100644 --- a/vidmap/mapper/stages/global_positioning/problem.py +++ b/vidmap/mapper/stages/global_positioning/problem.py @@ -17,15 +17,15 @@ @dataclass(frozen=True) class TemporalAccelerationPrior: - prev_image_id: int - image_id: int - next_image_id: int + prev_frame_id: int + frame_id: int + next_frame_id: int dt_prev: float dt_next: float sqrt_observation_count: float def __post_init__(self): - ids = (self.prev_image_id, self.image_id, self.next_image_id) + ids = (self.prev_frame_id, self.frame_id, self.next_frame_id) values = (self.dt_prev, self.dt_next, self.sqrt_observation_count) if len(set(ids)) != 3 or any(not np.isfinite(v) or v <= 0 for v in values): raise ValueError("invalid temporal acceleration prior") @@ -59,17 +59,17 @@ def _loss(config): return pyceres.LossFunction(dict(name=config.name, params=[config.scale], magnitude=config.weight)) -def prepare_observations(reconstruction, sidecars, options: GPOptions, image_timeline=()): +def prepare_observations(reconstruction, sidecars, options: GPOptions, frame_timeline=()): first = options.first_pass if first.sequential_support_warmup_rounds: - if len(image_timeline) != reconstruction.num_images() or set(image_timeline) != set(reconstruction.images): - raise ValueError("Sequential support requires an exact unique image timeline") + if len(frame_timeline) != reconstruction.num_frames() or set(frame_timeline) != set(reconstruction.frames): + raise ValueError("Sequential support requires an exact unique frame timeline") return gp_costs.prepare_observations( reconstruction, sidecars, options.track_filter.min_num_views_per_track, options.common.use_lc_observations, - image_timeline, + frame_timeline, first.sequential_support_observations_per_track if first.sequential_support_warmup_rounds else 0, ) @@ -123,9 +123,7 @@ def run_global_positioning( playback_options = playback_options or SolverPlaybackOptions() playback_options.validate() images = reconstruction.images - for image in images.values(): - if image.has_pose and not image.is_ref_in_frame(): - raise ValueError("VidMap global positioning requires reference cameras") + rig_scene = gp_costs.validate_fixed_rigs(reconstruction) # GP1 initializes positions; GP2 starts from the GP1 result. gp_options = pycolmap.GlobalPositionerOptions( @@ -141,6 +139,8 @@ def run_global_positioning( experimental_observation_stddev=pixel_stddev, uncalibrated_observation_weight=1.0, ) + if rig_scene: + gp_options.refine_sensor_from_rig = False normal_geometry = _loss( pass_options.loss_normal_geometry if options.common.use_metric_depth_constraint @@ -178,18 +178,15 @@ def run_global_positioning( positioner = pycolmap.create_default_global_positioner( gp_options, pycolmap.PoseGraph(), reconstruction, default_loss ) + gp_costs.collect_default_observations(positioner, reconstruction, selections[0], pixel_stddev) finally: gp_costs.swap_tracks(reconstruction, selected_tracks) del selected_tracks problem = positioner.problem - centers = { - image_id: positioner.frame_center_parameter_block(image.frame_id) - for image_id, image in images.items() - if image.has_pose - } - centers = {i: center for i, center in centers.items() if center is not None} - if len({images[image_id].frame_id for image_id in centers}) != len(centers): - raise ValueError("VidMap global positioning requires one image per frame") + # Retain first-image traversal order for deterministic appended initialization. + frames = {image.frame_id: image.frame for image in images.values() if image.has_pose} + frame_centers = {fid: positioner.frame_center_parameter_block(fid) for fid in frames} + frame_centers = {fid: center for fid, center in frame_centers.items() if center is not None} random = np.random.default_rng(common.random_seed if common.random_seed >= 0 else None) if gp_options.generate_random_points: gp_costs.initialize_points(problem, reconstruction, options.track_filter.min_num_views_per_track, random) @@ -199,17 +196,16 @@ def run_global_positioning( if capture_state: point_ids, points_xyz, _ = native.point3D_table(reconstruction) result.initial_point3D_xyz = dict(zip(point_ids.tolist(), points_xyz)) - gp_costs.collect_default_observations(problem, reconstruction, centers, selections[0]) extra_centers = {} - for image_id, image in reconstruction.images.items(): - if image_id in centers or not image.has_pose: + for frame_id, frame in frames.items(): + if frame_id in frame_centers: continue if gp_options.generate_random_positions: center = random.uniform(-100.0, 100.0, 3) else: - center = image.projection_center().copy() - extra_centers[image_id] = centers[image_id] = center + center = frame.rig_from_world.inverse().translation.copy() + extra_centers[frame_id] = frame_centers[frame_id] = center # Keep appended parameter storage alive through both solves. _appended_scales = [ @@ -217,7 +213,7 @@ def run_global_positioning( positioner, reconstruction, selection, - centers, + frame_centers, loss, pixel_stddev, gp_options.initialize_scales_from_geometry, @@ -226,7 +222,8 @@ def run_global_positioning( for selection, loss in zip(selections[1:], geometry_losses[1:]) if len(selection) ] - if not options.common.use_metric_depth_constraint and gp_options.optimize_scales: + + if not rig_scene and not options.common.use_metric_depth_constraint and gp_options.optimize_scales: gp_costs.fix_first_observation_scale(problem, selections) regular, loop, num_points, has_support = gp_costs.observation_counts(selections) if warmup_rounds and not has_support: @@ -295,25 +292,23 @@ def run_global_positioning( getattr(temporal, prefix + "_huber_width") / temporal.stddev, ) for prior in temporal_priors: + frame_ids = (prior.prev_frame_id, prior.frame_id, prior.next_frame_id) cost = gp_costs.temporal_acceleration_cost(prior.dt_prev, prior.dt_next, temporal.stddev) weighted = gp_costs.scaled_loss(loss, temporal_weight * prior.sqrt_observation_count**2) problem.add_residual_block( cost, weighted, - [centers[i] for i in (prior.prev_image_id, prior.image_id, prior.next_image_id)], + [frame_centers[fid] for fid in frame_ids], ) diagnostics.num_temporal_acceleration_residuals += 1 - for image_id in list(extra_centers): - if not problem.has_parameter_block(extra_centers[image_id]): - del extra_centers[image_id] - del centers[image_id] + frame_centers = {fid: center for fid, center in frame_centers.items() if problem.has_parameter_block(center)} if capture_state: - result.initial_frame_centers = {images[i].frame_id: center.copy() for i, center in centers.items()} + result.initial_frame_centers = {fid: center.copy() for fid, center in frame_centers.items()} # Configure parameter ordering, solver options and playback. positioner.extend_parameter_block_ordering( - [(block, 2) for arrays in (extra_centers, depth_scales) for block in arrays.values()] + [(block, 2) for arrays in (frame_centers, depth_scales) for block in arrays.values()] ) solver_options = pyceres.SolverOptions(positioner.solver_options) apply_solver_backend(solver_options, options.solver_backend) @@ -324,7 +319,7 @@ def run_global_positioning( playback_callback, playback_options, reconstruction, - centers, + frame_centers, selections, geometry_losses, problem, @@ -353,27 +348,19 @@ def run_global_positioning( diagnostics.update(summary) capture.emit("final", warmup_rounds + diagnostics.num_iterations - 1) - # Write optimized centers back to camera poses. - # Finalize also writes factory centers whose observations were all rejected. - inactive_translations = { - image.frame_id: image.frame.rig_from_world.translation.copy() - for image_id, image in images.items() - if image.has_pose and image_id not in centers - } if not positioner.finalize(summary): raise RuntimeError("Global positioning finalization failed") - for frame_id, translation in inactive_translations.items(): - reconstruction.frame(frame_id).rig_from_world.translation = translation - for image_id, center in extra_centers.items(): - pose = reconstruction.frame(images[image_id].frame_id).rig_from_world - pose.translation = -(pose.rotation * center) + for fid, center in extra_centers.items(): + if fid in frame_centers: + pose = reconstruction.frame(fid).rig_from_world + pose.translation = -(pose.rotation * center) result.success = True result.depth_map_scales = { i: float(np.exp(v[0]) if common.use_log_scale_for_depth_map_scales else v[0]) for i, v in depth_scales.items() } diagnostics.num_depth_map_scales = len(depth_scales) - diagnostics.num_camera_centers = len(centers) + diagnostics.num_camera_centers = len(frame_centers) diagnostics.num_point3D_parameters = num_points del capture, problem, positioner return result @@ -397,18 +384,23 @@ def _scale_values(selections): class _Playback(pyceres.IterationCallback): - def __init__(self, callback, options, reconstruction, centers, observations, losses, problem): + def __init__(self, callback, options, reconstruction, frame_centers, observations, losses, problem): super().__init__() self.callback, self.options = callback, options - self.centers = centers + self.frame_centers = frame_centers self.reconstruction = reconstruction self.observations = observations self.losses = losses if callback is None: return - self.image_ids = list(options.image_ids) or sorted(centers) + self.image_ids = list(options.image_ids) or sorted( + i for i, image in reconstruction.images.items() if image.frame_id in frame_centers + ) self.point_ids = gp_costs.playback_point_ids(problem, reconstruction, list(options.point3D_ids), 200000) - if any(i not in centers for i in self.image_ids): + if any( + i not in reconstruction.images or reconstruction.image(i).frame_id not in frame_centers + for i in self.image_ids + ): raise ValueError("Playback image is not active in global positioning") self.offset = 0 @@ -425,7 +417,7 @@ def emit(self, phase, iteration): image_ids = self.image_ids point_ids = self.point_ids camera_values, point_values = native.playback_coordinates( - self.reconstruction, image_ids, point_ids, self.centers + self.reconstruction, image_ids, point_ids, self.frame_centers ) self.callback( dict( diff --git a/vidmap/mapper/stages/relative_pose/estimator.py b/vidmap/mapper/stages/relative_pose/estimator.py index 51c3085..a3c1117 100644 --- a/vidmap/mapper/stages/relative_pose/estimator.py +++ b/vidmap/mapper/stages/relative_pose/estimator.py @@ -26,6 +26,23 @@ logger = logging.getLogger(__name__) +def _fixed_stereo_pose(rec, first, second): + """Use calibrated sensor poses for two simultaneous views of one frame.""" + if first.camera_id == second.camera_id: + raise ValueError("Same-frame relative pose requires distinct calibrated sensors") + rig = rec.rigs[rec.frames[first.frame_id].rig_id] + poses = [] + for image in (first, second): + camera = rec.cameras[image.camera_id] + if not camera.has_prior_focal_length: + raise ValueError("Same-frame relative pose requires fixed calibrated intrinsics") + pose = pycolmap.Rigid3d() if rig.is_ref_sensor(camera.sensor_id) else rig.sensor_from_rig(camera.sensor_id) + if pose is None or not np.isfinite(pose.matrix()).all() or abs(np.linalg.norm(pose.rotation.quat) - 1) > 1e-8: + raise ValueError("Same-frame relative pose requires finite fixed sensor transforms") + poses.append(pose) + return poses[1] * poses[0].inverse() + + @dataclass(frozen=True) class RelativePoseResult: inlier_thresholds: InlierThresholdOptions @@ -45,10 +62,26 @@ def estimate(self) -> RelativePoseResult: rec = self.solve_state.reconstruction cameras = rec.cameras images = rec.images - consecutive_pair_ids = self.consecutive_pair_ids + consecutive_pair_ids = set(self.consecutive_pair_ids) inlier_thresholds = self.inlier_threshold_options native_inlier_thresholds = build_inlier_threshold_options(inlier_thresholds) filtered_consecutive_pairs = set() + valid_pair_ids = [] + fixed_poses = {} + for pair_id in tqdm( + state.pair_order, + desc="Filtering valid image pairs", + disable=not progress_bars_enabled(), + ): + if not state.pose_graph.is_valid(pair_id): + continue + valid_pair_ids.append(pair_id) + first_id, second_id = pycolmap.pair_id_to_image_pair(pair_id) + first, second = images[first_id], images[second_id] + if first.frame_id == second.frame_id: + if pair_id in consecutive_pair_ids: + raise ValueError("consecutive_pair_ids must contain temporal pairs, not same-frame stereo") + fixed_poses[pair_id] = _fixed_stereo_pose(rec, first, second) identity_pose = pycolmap.Rigid3d() for pair_id, edge in state.pose_graph.edges.items(): edge.cam2_from_cam1 = identity_pose @@ -75,14 +108,6 @@ def estimate(self) -> RelativePoseResult: for image_id, image in images.items() } - valid_pair_ids = [] - for image_pair_id in tqdm( - state.pair_order, - desc="Filtering valid image pairs", - disable=not progress_bars_enabled(), - ): - if state.pose_graph.is_valid(image_pair_id): - valid_pair_ids.append(image_pair_id) logger.info("Estimating relative poses for %d image pairs", len(valid_pair_ids)) write_replay = self.replay.write_enabled("relative_pose") @@ -97,8 +122,18 @@ def estimate(self) -> RelativePoseResult: image_valid_cache=image_valid_cache, ) pair_args_list = [] + results: dict[int, MDRPResult] = {} for image_pair_id in valid_pair_ids: image_pair = state.pair_data(image_pair_id) + if image_pair_id in fixed_poses: + results[image_pair_id] = { + "is_valid": True, + "cam2_from_cam1": fixed_poses[image_pair_id], + "inliers": np.empty(0, dtype=np.int32), + # No depth-scale fit: retain the neutral replay-only value. + "rel_depth_scale": 1.0, + } + continue pair_args_list.append( ( image_pair_id, @@ -107,7 +142,6 @@ def estimate(self) -> RelativePoseResult: ) ) - results: dict[int, MDRPResult] = {} if self.options.max_workers == 1: for pair_args in tqdm( pair_args_list, @@ -177,6 +211,12 @@ def estimate(self) -> RelativePoseResult: ) filtered_consecutive_pairs = current + if write_replay: + for pair_id in fixed_poses: + results[pair_id]["inliers"] = np.asarray( + state.pair_data(pair_id).inlier_indices, dtype=np.int32 + ).copy() + summary = ( relative_pose_summary( state, diff --git a/vidmap/mapper/stages/rotation_averaging.py b/vidmap/mapper/stages/rotation_averaging.py index 139887b..b054730 100644 --- a/vidmap/mapper/stages/rotation_averaging.py +++ b/vidmap/mapper/stages/rotation_averaging.py @@ -9,6 +9,7 @@ import pyceres import pycolmap from scipy.cluster.hierarchy import DisjointSet +from vidmap_native import global_positioning as gp_costs from vidmap.mapper.native.extension import native from vidmap.mapper.native.state import SolveState @@ -20,6 +21,18 @@ logger = logging.getLogger(__name__) +def add_fixed_rotation_sensors(problem, reconstruction, sensors): + for rig_id, camera_id in sensors: + rig = reconstruction.rig(rig_id) + sensor_id = reconstruction.camera(camera_id).sensor_id + if sensor_id == rig.ref_sensor_id: + continue + rotation = rig.non_ref_sensors[sensor_id].rotation.quat + if not problem.has_parameter_block(rotation): + problem.add_parameter_block(rotation, 4) + problem.set_parameter_block_constant(rotation) + + @dataclass(kw_only=True) class RotationAverager: solve_state: SolveState @@ -30,6 +43,7 @@ class RotationAverager: def run_pass(self) -> bool: rec = self.solve_state.reconstruction + gp_costs.validate_fixed_rigs(rec) graph, tracking_graph = self.build_graphs() active_frames = graph.largest_connected_frame_component(rec, self.options.filter_unregistered_images) if not active_frames: @@ -38,6 +52,7 @@ def run_pass(self) -> bool: graph.invalidate_pairs_outside_active_image_ids(active_images) tracking_graph.invalidate_pairs_outside_active_image_ids(active_images) self.bridge_tracking_gaps(graph, tracking_graph, active_frames) + tracking_graph.add_rig_edges(rec, active_images, refine_sensor_from_rig=False) keep_frames = {fid for fid, frame in rec.frames.items() if frame.has_pose()} if not self.solve(graph, tracking_graph): return False @@ -51,6 +66,9 @@ def build_graphs(self) -> tuple[pycolmap.PoseGraph, pycolmap.PoseGraph]: pair = state.pair_data(pair_id) if not edge.valid or not pair.has_relative_pose: continue + image_id1, image_id2 = pycolmap.pair_id_to_image_pair(pair_id) + if state.image(image_id1).frame_id == state.image(image_id2).frame_id: + continue inliers = pair.inlier_indices is_tracking = len(inliers) > 0 and 2 * np.count_nonzero(pair.are_loop_closure[inliers]) <= len(inliers) graph.edges[pair_id] = edge @@ -89,18 +107,26 @@ def temporal_gap(pair_id): def solve(self, graph: pycolmap.PoseGraph, tracking_graph: pycolmap.PoseGraph) -> bool: options = pycolmap.RotationEstimatorOptions() options.ceres.loss_function_scale = np.rad2deg(self.options.video_tracking_huber_scale) + options.refine_sensor_from_rig = False options.ceres.solver_options.max_num_iterations = self.options.max_iterations options.ceres.solver_options.num_threads = 1 if self.options.num_threads is None else self.options.num_threads loop_loss = pyceres.CauchyLoss(self.options.video_lc_cauchy_scale) + loop_edges = { + pair_id: edge + for pair_id, edge in graph.edges.items() + if edge.valid and pair_id not in tracking_graph.edges and not self.options.filter_risky_loop_closure_pairs + } + rec = self.solve_state.reconstruction + loop_images = {i for pair_id in loop_edges for i in pycolmap.pair_id_to_image_pair(pair_id)} + loop_sensors = {(rec.image(i).frame.rig_id, rec.image(i).camera_id) for i in loop_images} # Warm up with Huber on tracking edges, then refine with Cauchy on all edges. for loss_type in (pycolmap.LossFunctionType.HUBER, pycolmap.LossFunctionType.CAUCHY): options.ceres.loss_function_type = loss_type averager = pycolmap.create_default_ceres_rotation_averager( options, tracking_graph, self.solve_state.reconstruction ) - for pair_id, edge in graph.edges.items(): - if not edge.valid or pair_id in tracking_graph.edges or self.options.filter_risky_loop_closure_pairs: - continue + add_fixed_rotation_sensors(averager.problem, rec, loop_sensors) + for pair_id, edge in loop_edges.items(): image_id1, image_id2 = pycolmap.pair_id_to_image_pair(pair_id) averager.add_relative_rotation_residual(image_id1, image_id2, edge.cam2_from_cam1.rotation, loop_loss) if not averager.solve().IsSolutionUsable(): diff --git a/vidmap/reconstruction.py b/vidmap/reconstruction.py index fa93795..b855d2e 100644 --- a/vidmap/reconstruction.py +++ b/vidmap/reconstruction.py @@ -15,7 +15,7 @@ from PIL import Image -from vidmap.datasets.local import IMAGE_SUFFIXES, LocalImageParser +from vidmap.datasets.local import IMAGE_SUFFIXES from vidmap.mapper import Mapper from vidmap.mapper.inputs import MapperInputs from vidmap.run_options import RunOptions @@ -33,49 +33,6 @@ _LOCAL_INPUT_SCHEMA_VERSION = 1 -def reconstruct( - mapping_conf, - frontend_conf, - input_path: str | Path, - *, - workspace: str | Path, - mapper_inputs_dir: str | Path | MapperInputs, - run_options: RunOptions | None = None, - imnames=None, - intrinsics_path: str | Path | None = None, - overwrite_outputs: bool = False, - output_dir: str | Path | None = None, -): - """Reconstruct an image directory or MP4 using the shared mapping API.""" - mapper_inputs = resolve_mapper_inputs(frontend_conf, mapper_inputs_dir) - estimate_intrinsics = frontend_conf.pipeline.camera_priors.initialization == "predicted" - if estimate_intrinsics and intrinsics_path is not None: - raise ValueError("The selected uncalibrated config cannot be combined with --intrinsics") - if not estimate_intrinsics and intrinsics_path is None: - raise ValueError("The selected calibrated config requires --intrinsics") - - workspace = Path(workspace).expanduser() - image_dir = prepare_reconstruction_images(input_path, workspace) - scene_parser = LocalImageParser( - image_dir=image_dir, - imnames=imnames, - intrinsics_path=intrinsics_path, - estimate_intrinsics=estimate_intrinsics, - ) - output_dir = workspace if output_dir is None else Path(output_dir).expanduser() - run_options = RunOptions() if run_options is None else run_options - return run_mapping( - mapping_conf, - frontend_conf=frontend_conf, - mapper_inputs=mapper_inputs, - run_options=run_options, - scene_parser=scene_parser, - output_dir=output_dir, - scene_name=scene_parser.scene, - overwrite_outputs=overwrite_outputs, - ) - - def resolve_mapper_inputs(frontend_conf, mapper_inputs: str | Path | MapperInputs) -> MapperInputs: """Attach the expected frontend identity without hashing mapper payloads.""" from vidmap.frontend.identity import frontend_config_identity @@ -117,7 +74,12 @@ def prepare_reconstruction_images(input_path: str | Path, workspace: str | Path) return decode_video_frames(source, Path(workspace).expanduser()) -def write_local_input_provenance(directory: str | Path, image_dir: str | Path, *, overwrite: bool = False) -> Path: +def write_local_input_provenance( + directory: str | Path, + image_dir: str | Path, + *, + overwrite: bool = False, +) -> Path: """Record the resolved RGB root needed to visualize a local reconstruction.""" directory = Path(directory).expanduser() image_dir = Path(image_dir).expanduser().resolve() @@ -130,7 +92,7 @@ def write_local_input_provenance(directory: str | Path, image_dir: str | Path, * "image_dir": str(image_dir), } if path.is_file() and not overwrite: - if _read_local_input_provenance(path) != image_dir: + if _read_local_input_provenance(path) != payload: raise ValueError(f"Local input provenance already identifies different images: {path}") return path directory.mkdir(parents=True, exist_ok=True) @@ -144,12 +106,18 @@ def write_local_input_provenance(directory: str | Path, image_dir: str | Path, * def local_input_image_dir(mapper_inputs: str | Path | MapperInputs) -> Path | None: - """Return the RGB root recorded in a mapper-inputs boundary, when present.""" + """Return recorded reconstruction images.""" directory = ( Path(mapper_inputs).expanduser() if isinstance(mapper_inputs, (str, Path)) else Path(mapper_inputs.directory) ) source = directory / LOCAL_INPUT_MANIFEST_NAME - return _read_local_input_provenance(source) if source.is_file() else None + if not source.is_file(): + return None + payload = _read_local_input_provenance(source) + root = Path(payload["image_dir"]).expanduser() + if not root.is_dir(): + raise FileNotFoundError(f"Recorded local input image directory is missing: {root}") + return root def extract_point_colors( @@ -187,7 +155,7 @@ def local_run_image_dir(run: str | Path) -> Path | None: return local_input_image_dir(Path(run).expanduser() / "mapper_inputs") -def _read_local_input_provenance(path: Path) -> Path: +def _read_local_input_provenance(path: Path) -> dict: try: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as error: @@ -201,10 +169,7 @@ def _read_local_input_provenance(path: Path) -> Path: or not payload["image_dir"] ): raise ValueError(f"Invalid local input provenance: {path}") - image_dir = Path(payload["image_dir"]).expanduser() - if not image_dir.is_dir(): - raise FileNotFoundError(f"Recorded local input image directory is missing: {image_dir}") - return image_dir + return payload def decode_video_frames(video_path: str | Path, workspace: str | Path) -> Path: diff --git a/vidmap/run.py b/vidmap/run.py index c1418ae..a979e6e 100644 --- a/vidmap/run.py +++ b/vidmap/run.py @@ -24,7 +24,6 @@ def build_parser() -> ArgumentParser: help="Run directory for resolved configs, frontend artifacts, decoded frames, and reconstruction outputs.", ) parser.add_argument("--imnames", nargs="*", type=str) - parser.add_argument("--intrinsics", type=str) parser.add_argument("--name", type=str) parser.add_argument("--force-frontend", action="store_true") parser.add_argument( @@ -84,27 +83,22 @@ def main(argv=None): args.input_data, workspace=args.output, imnames=args.imnames, - intrinsics_path=args.intrinsics, force_frontend=args.force_frontend, cache_depth_maps=args.cache_depth_maps, device=run_options.device, ) logger.info("Mapper inputs ready: tag=%s path=%s", frontend.tag, frontend.path) - from vidmap.reconstruction import reconstruct + from vidmap.reconstruction import run_mapping output_dir = Path(args.output).expanduser() - reconstruction = reconstruct( + reconstruction = run_mapping( mapping_conf, - frontend_conf, - args.input_data, - workspace=args.output, - mapper_inputs_dir=frontend.mapper_inputs, + frontend_conf=frontend_conf, + mapper_inputs=frontend.mapper_inputs, + output_dir=output_dir, run_options=run_options, - imnames=args.imnames, - intrinsics_path=args.intrinsics, overwrite_outputs=args.overwrite, - output_dir=output_dir, ) reconstruction_dir = output_dir / "rec" reconstruction_dir.mkdir(parents=True, exist_ok=True) diff --git a/vidmap/runtime_variants.py b/vidmap/runtime_variants.py index 2587d65..ab75761 100644 --- a/vidmap/runtime_variants.py +++ b/vidmap/runtime_variants.py @@ -17,7 +17,7 @@ class ColmapRuntimeVariant: COLMAP_RUNTIME_VARIANTS = { "stock": ColmapRuntimeVariant( name="stock", - revision="9e653da3baaaada926bda6c5286568413fb3e675", + revision="b1972bc226f58e095c8b75bc214f4bb2d8ba57e4", pycolmap_major_minor=(4, 3), canonical_environment="VIDMAP_STOCK_COLMAP", ), diff --git a/vidmap/visualization/html/browser/browser_reconstruction.js b/vidmap/visualization/html/browser/browser_reconstruction.js index 772ee47..32d0275 100644 --- a/vidmap/visualization/html/browser/browser_reconstruction.js +++ b/vidmap/visualization/html/browser/browser_reconstruction.js @@ -13,6 +13,32 @@ return error instanceof Error ? error.message : String(error); } + function parseMapperManifest(text) { + const manifest = JSON.parse(text); + if (manifest.schema_version !== 14 || manifest.frame_sequence == null) { + throw new Error("Expected current mapper-input manifest with frame_sequence"); + } + const times = manifest.frame_sequence.timestamps_ns; + if (times === null) return manifest; + if (!Array.isArray(times)) throw new Error("Invalid mapper frame timestamps"); + // Match whole JSON strings first, so digits inside names/escaped strings stay untouched. + // A second parse preserves integer tokens beyond Number's exact range without changing + // the external manifest schema or requiring JSON.parse's newer source-context API. + const exact = JSON.parse(text.replace(/"(?:\\[\s\S]|[^"\\])*"|-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/g, + token => /^-?\d+$/.test(token) && !Number.isSafeInteger(Number(token)) ? JSON.stringify(token) : token)); + for (const [id, value] of times.entries()) { + if (typeof value !== "number" || !Number.isFinite(value) || !Number.isInteger(value)) { + throw new Error("Invalid mapper frame timestamp"); + } + const integer = exact.frame_sequence.timestamps_ns[id]; + if (typeof integer === "number" && !Number.isSafeInteger(integer)) { + throw new Error("Nanosecond timestamps must use exact integer JSON tokens"); + } + times[id] = BigInt(integer); + } + return manifest; + } + function workerRequest(command, payload, transferables = [], onProgress = null) { return new Promise((resolve, reject) => { const sourceUrl = URL.createObjectURL( @@ -91,6 +117,8 @@ imagePreviews: options.imagePreviews ?? null, localInput: options.localInput ?? null, covarianceCache: options.covarianceCache ?? null, + framesFile: await optionalFile(getFile, "rec/frames.bin"), + captureManifest: await optionalFile(getFile, "mapper_inputs/manifest.json"), database: database, loopClosureMasks: loopClosureMasks }; @@ -157,7 +185,11 @@ if ((databaseEntry === undefined) !== (masksEntry === undefined)) { throw new Error("Embedded run contains incomplete loop-closure data"); } - return selectionFrom(path => decodedEmbeddedFile(payload.files[path], path), { + const getFile = async path => { + if (payload.files[path] === undefined) throw new DOMException("Missing embedded file", "NotFoundError"); + return decodedEmbeddedFile(payload.files[path], path); + }; + return selectionFrom(getFile, { imagePreviews: payload.imagePreviews, covarianceCache, database: databaseEntry === undefined ? null : await decodedEmbeddedFile(databaseEntry, DATABASE_PATH), @@ -260,23 +292,27 @@ } } - function databaseEdgeSharedCounts(databasePairs, observationPointIds, observationOffsets) { + function frameObservationPointIds(imageIndices, observationPointIds, observationOffsets) { const invalidPointId = 0xffffffffffffffffn; - function observedPointIds(keyframe) { - const ids = new Set(); + const ids = new Set(); + for (const imageIndex of imageIndices) { + if (imageIndex === null) continue; for ( - let observation = observationOffsets[keyframe]; - observation < observationOffsets[keyframe + 1]; + let observation = observationOffsets[imageIndex]; + observation < observationOffsets[imageIndex + 1]; ++observation ) { const pointId = observationPointIds[observation]; if (pointId !== invalidPointId) ids.add(pointId); } - return ids; } + return ids; + } + + function databaseEdgeSharedCounts(databasePairs, observationPointIds, observationOffsets) { const result = new Uint32Array(databasePairs.length / 2); for (let index = 0; index < result.length; ++index) { - const leftIds = observedPointIds(databasePairs[index * 2]); + const leftIds = frameObservationPointIds([databasePairs[index * 2]], observationPointIds, observationOffsets); let shared = 0; const right = databasePairs[index * 2 + 1]; for (let observation = observationOffsets[right]; observation < observationOffsets[right + 1]; ++observation) { @@ -625,6 +661,8 @@ keyframeTimeline.imageIds[rightKeyframe] ); showLoopClosurePair({ + leftImageIndex: leftKeyframe, + rightImageIndex: rightKeyframe, leftName: leftName, rightName: rightName, leftKeypoints: matches.leftKeypoints, @@ -653,13 +691,17 @@ minimumTrackLength.value = "1"; minimumTrackLengthLabel.textContent = "1 observation"; setStatus("reading COLMAP model…"); + let database = null; try { const files = selection.files; const buffers = await sourceBuffers(files); const transferables = COLMAP_FILENAMES.map(name => buffers[name]); + const framesBuffer = selection.framesFile === null ? null : await selection.framesFile.arrayBuffer(); + const manifest = selection.captureManifest === null ? null : parseMapperManifest(await selection.captureManifest.text()); + if (framesBuffer !== null) transferables.push(framesBuffer); const loaded = await workerRequest( "load", - {buffers: buffers}, + {buffers: buffers, framesBuffer, frameSequence: manifest?.frame_sequence ?? null}, transferables ); installKeyframeTimeline(loaded.timeline); @@ -675,12 +717,13 @@ observationPositions: loaded.cameras.observationPositions, observationPointIds: loaded.cameras.observationPointIds, observationOffsets: loaded.cameras.observationOffsets, - database: null, + database: database, loopClosurePairs: null, ranks: null, eligibleCount: 0 }; replacePointGeometry(state.positions, loaded.points.colors); + installPointIds(state.pointIds); installTrackLengths(loaded.points.trackLengths); let maximumTrackLength = 1; for (const length of loaded.points.trackLengths) maximumTrackLength = Math.max(maximumTrackLength, length); @@ -696,7 +739,8 @@ setStatus("model loaded; validating loop-closure masks…"); const loopClosurePairs = parseLoopClosureMasks(await selection.loopClosureMasks.text()); setStatus("loop-closure masks validated; reading verified database edges…"); - state.database = await openDatabase(selection.database); + if (database === null) database = await openDatabase(selection.database); + state.database = database; state.loopClosurePairs = loopClosurePairs; const databasePairs = databaseEdgeKeyframes( state.database, @@ -721,15 +765,15 @@ state.observationPointIds, state.observationOffsets ); - showKeyframe(Number(keyframeSlider.value)); if ( selection.imagePreviews !== null && Object.keys(selection.imagePreviews).length > 0 ) { - installImageDirectory(selection.imagePreviews); + imageDirectory = selection.imagePreviews; } else if (selection.localInput !== null) { - installImageDirectory(parseLocalInput(await selection.localInput.text())); + imageDirectory = parseLocalInput(await selection.localInput.text()); } + showKeyframe(Number(keyframeSlider.value)); setStatus(`${state.positions.length / 3} points loaded; checking covariance cache…`); let cacheBuffer = null; if (selection.covarianceCache !== null) { @@ -756,7 +800,7 @@ } } } catch (error) { - if (state !== null && state.database !== null) state.database.close(); + if (database !== null) database.close(); state = null; resetRunVisualState(); reportError(errorMessage(error)); @@ -879,11 +923,14 @@ if (typeof module !== "undefined" && module.exports) { module.exports = { + parseMapperManifest: parseMapperManifest, matchingFingerprints: matchingFingerprints, + frameObservationPointIds: frameObservationPointIds, parseCovarianceCache: parseCovarianceCache, covarianceCacheBuffer: covarianceCacheBuffer, parseLocalInput: parseLocalInput, selectionFrom: selectionFrom, + databaseEdgeKeyframes: databaseEdgeKeyframes, databaseEdgeSharedCounts: databaseEdgeSharedCounts, sharedTrackObservations: sharedTrackObservations, loopClosureMatchObservations: loopClosureMatchObservations, diff --git a/vidmap/visualization/html/browser/browser_scene.js b/vidmap/visualization/html/browser/browser_scene.js index de22edc..ab21b37 100644 --- a/vidmap/visualization/html/browser/browser_scene.js +++ b/vidmap/visualization/html/browser/browser_scene.js @@ -148,7 +148,7 @@ __RECONSTRUCTION_SCRIPT__ }); let pointPositions = new Float32Array(); let pointColors = new Uint8Array(); - const keyframeTimeline = {names: [], pointOffsets: [0], imageIds: [], timestampsSeconds: null}; + const keyframeTimeline = {names: [], pointOffsets: [0], imageIds: [], timestampsSeconds: null, frames: [], frameImageOffsets: [0]}; let imageDirectory = null; let estimatedFrustaPositions = new Float32Array(); let estimatedPathPositions = new Float32Array(); @@ -324,16 +324,23 @@ __RECONSTRUCTION_SCRIPT__ return {width: width, height: height}; } - function drawTrackedKeypoints(index) { - const canvas = document.getElementById("tracked-keypoints-overlay"); - const preview = document.getElementById("keyframe-preview"); - const width = preview.clientWidth; - const height = preview.clientHeight; + function overlayContext(image, canvas) { const pixelRatio = window.devicePixelRatio || 1; - canvas.width = Math.max(1, Math.round(width * pixelRatio)); - canvas.height = Math.max(1, Math.round(height * pixelRatio)); + canvas.width = Math.max(1, Math.round(image.clientWidth * pixelRatio)); + canvas.height = Math.max(1, Math.round(image.clientHeight * pixelRatio)); const drawing = canvas.getContext("2d"); drawing.clearRect(0, 0, canvas.width, canvas.height); + const size = imageDimensions(image); + const scale = Math.min(canvas.width / size.width, canvas.height / size.height); + const width = size.width * scale, height = size.height * scale; + const x = (canvas.width - width) / 2, y = (canvas.height - height) / 2; + drawing.beginPath(); drawing.rect(x, y, width, height); drawing.clip(); + drawing.translate(x, y); + return {drawing, pixelRatio, scaleX: scale, scaleY: scale}; + } + + function drawTrackedImageKeypoints(index, preview, canvas) { + const {drawing, pixelRatio, scaleX, scaleY} = overlayContext(preview, canvas); const toggle = document.getElementById("tracked-keypoints-toggle"); if ( trackedKeypointPositions === null || trackedKeypointOffsets === null || @@ -342,9 +349,6 @@ __RECONSTRUCTION_SCRIPT__ ) return; const begin = trackedKeypointOffsets[index]; const end = trackedKeypointOffsets[index + 1]; - const sourceDimensions = imageDimensions(preview); - const scaleX = canvas.width / sourceDimensions.width; - const scaleY = canvas.height / sourceDimensions.height; const radius = Math.max(1.7 * pixelRatio, 1); drawing.fillStyle = "rgba(0, 255, 170, 0.9)"; drawing.strokeStyle = "rgba(0, 0, 0, 0.85)"; @@ -360,6 +364,111 @@ __RECONSTRUCTION_SCRIPT__ } } + let previewGeneration = 0; + let framePreviewCards = []; + function drawTrackedKeypoints() { + for (const card of framePreviewCards) { + if (card.stage.hidden) continue; + if (card.imageIndex !== null) drawTrackedImageKeypoints(card.imageIndex, card.image, card.canvas); + } + } + + async function showFramePreviews(frameIndex) { + const generation = ++previewGeneration; + const frame = keyframeTimeline.frames[frameIndex]; + const container = document.getElementById("keyframe-previews"); + if (frame === undefined) return; + container.dataset.loading = "true"; + const directory = imageDirectory; + const cards = frame.cameraIds.map((cameraId, slot) => { + const figure = document.importNode(document.getElementById("keyframe-card").content.firstElementChild, true); + const caption = figure.querySelector("figcaption"), stage = figure.querySelector("div"); + const image = figure.querySelector("img"), canvas = figure.querySelector("canvas"); + figure.className = "keyframe-camera"; + figure.dataset.cameraId = String(cameraId); + const imageIndex = frame.imageIndices[slot]; + const name = imageIndex === null ? null : keyframeTimeline.names[imageIndex]; + caption.textContent = `Camera ${cameraId} Β· ${name ?? "not registered in this frame"}`; + image.alt = `Selected frame camera ${cameraId}`; + image.dataset.imageIndex = imageIndex === null ? "" : String(imageIndex); + stage.hidden = name === null || directory === null; + if (name !== null && directory === null) caption.textContent += " Β· preview unavailable"; + return {figure, image, canvas, imageIndex, stage, name}; + }); + // Decode every camera before revealing any image. A newer slider change + // invalidates the entire pending frame, never just one camera's request. + await Promise.all(cards.map(async card => { + if (card.stage.hidden) return; + try { + loadImage(card.image, directory, card.name); + await card.image.decode(); + } catch (failure) { + card.stage.hidden = true; + card.figure.querySelector("figcaption").textContent += ` Β· preview failed: ${errorMessage(failure)}`; + } + })); + if (generation !== previewGeneration) return; + container.replaceChildren(...cards.map(card => card.figure)); + container.dataset.frameIndex = String(frameIndex); + framePreviewCards = cards; + container.dataset.loading = "false"; + drawTrackedKeypoints(); + return true; + } + + const selectedFrameGeometry = pointGeometry(new Float32Array()); + const selectedFramePoints = new THREE.Points(selectedFrameGeometry, new THREE.PointsMaterial({ + color: 0x00ffaa, size: 3.5, sizeAttenuation: false, depthTest: false, depthWrite: false + })); + selectedFramePoints.name = "selected-frame-observations"; + selectedFramePoints.renderOrder = 105; + scene.add(selectedFramePoints); + let displayedPointIndex = new Map(); + function installPointIds(ids) { + displayedPointIndex = new Map(); + for (let index = 0; index < ids.length; ++index) displayedPointIndex.set(ids[index], index); + } + function updateSelectedFrameObservations(frameIndex) { + const frame = keyframeTimeline.frames[frameIndex]; + const positions = []; + if (frame !== undefined && trackedKeypointPointIds !== null) { + const ids = frameObservationPointIds(frame.imageIndices, trackedKeypointPointIds, trackedKeypointOffsets); + for (const id of ids) { + const index = displayedPointIndex.get(id); + if (index !== undefined) positions.push(...pointPositions.subarray(index * 3, index * 3 + 3)); + } + } + selectedFrameGeometry.setAttribute("position", new THREE.Float32BufferAttribute(positions, 3)); + selectedFrameGeometry.computeBoundingSphere(); + selectedFramePoints.visible = document.getElementById("frame-observations-toggle").checked; + return positions.length / 3; + } + + function updateFramePath(frameIndex) { + if (!keyframeTimeline.frames.some(frame => frame.cameraIds.length > 1)) { + updatePathPrefix(estimatedPath, estimatedPathPositions, keyframeTimeline.frameImageOffsets[frameIndex + 1]); + return; + } + const previous = new Map(); + const positions = []; + for (const frame of keyframeTimeline.frames.slice(0, frameIndex + 1)) { + frame.cameraIds.forEach((cameraId, slot) => { + const key = `${frame.rigId}:${cameraId}`; + const index = frame.imageIndices[slot]; + if (index === null) { previous.delete(key); return; } + const center = estimatedPathPositions.subarray(index * 3, index * 3 + 3); + if (previous.has(key)) positions.push(...previous.get(key), ...center); + previous.set(key, center); + }); + } + const geometry = new THREE.LineSegmentsGeometry(); + if (positions.length > 0) geometry.setPositions(positions); + estimatedPath.geometry.dispose(); + estimatedPath.geometry = geometry; + estimatedPath.visible = positions.length > 0 && document.getElementById("paths-toggle").checked; + invalidateSceneRender(); + } + function installTrackedKeypoints(positions, pointIds, offsets) { if (offsets.length !== keyframeTimeline.names.length + 1) { throw new Error("Tracked-keypoint offsets must match the keyframe timeline"); @@ -370,7 +479,7 @@ __RECONSTRUCTION_SCRIPT__ trackedKeypointPositions = positions; trackedKeypointPointIds = pointIds; trackedKeypointOffsets = offsets; - drawTrackedKeypoints(Number(document.getElementById("keyframe-slider").value)); + drawTrackedKeypoints(); } function installKeyframeTimeline(timeline) { @@ -384,11 +493,16 @@ __RECONSTRUCTION_SCRIPT__ keyframeTimeline.names = timeline.names; keyframeTimeline.imageIds = timeline.imageIds; keyframeTimeline.timestampsSeconds = timeline.timestampsSeconds; + keyframeTimeline.frames = timeline.frames; + keyframeTimeline.frameImageOffsets = timeline.frameImageOffsets; + document.getElementById("keyframe-preview-panel").classList.toggle("rig-frame", timeline.frames.some(f => f.cameraIds.length > 1)); keyframeTimeline.pointOffsets = new Array(timeline.names.length + 1).fill(0); const slider = document.getElementById("keyframe-slider"); - slider.max = String(timeline.names.length - 1); + slider.max = String(timeline.frames.length - 1); slider.value = slider.max; slider.disabled = false; + document.getElementById("keyframe-previous").disabled = timeline.frames.length < 2; + document.getElementById("keyframe-next").disabled = timeline.frames.length < 2; const play = document.getElementById("keyframe-play"); const status = document.getElementById("keyframe-playback-status"); play.disabled = timeline.timestampsSeconds === null; @@ -397,14 +511,7 @@ __RECONSTRUCTION_SCRIPT__ function drawLoopClosureKeypoints(image, canvas, keypoints, matchClasses) { if (!image.complete || !(image.naturalWidth > 0) || !(image.naturalHeight > 0)) return; - const pixelRatio = window.devicePixelRatio || 1; - canvas.width = Math.max(1, Math.round(image.clientWidth * pixelRatio)); - canvas.height = Math.max(1, Math.round(image.clientHeight * pixelRatio)); - const drawing = canvas.getContext("2d"); - drawing.clearRect(0, 0, canvas.width, canvas.height); - const sourceDimensions = imageDimensions(image); - const scaleX = canvas.width / sourceDimensions.width; - const scaleY = canvas.height / sourceDimensions.height; + const {drawing, pixelRatio, scaleX, scaleY} = overlayContext(image, canvas); const radius = Math.max(2 * pixelRatio, 1); drawing.strokeStyle = "rgba(0, 0, 0, 0.9)"; drawing.lineWidth = Math.max(pixelRatio, 1); @@ -437,7 +544,9 @@ __RECONSTRUCTION_SCRIPT__ } function showLoopClosurePair(selection) { - if (imageDirectory === null) throw new Error("Load the reconstruction image source before opening a pair"); + if (imageDirectory === null) { + throw new Error("Load the reconstruction image source before opening a pair"); + } const panel = document.getElementById("loop-closure-pair-panel"); activeLoopClosurePair = selection; const leftImage = document.getElementById("loop-closure-left-image"); @@ -474,12 +583,6 @@ __RECONSTRUCTION_SCRIPT__ setLoopClosurePointHighlights(new Float32Array()); } - function installImageDirectory(directory) { - imageDirectory = directory; - const slider = document.getElementById("keyframe-slider"); - showKeyframe(Number(slider.value)); - } - function resetRunVisualState() { imageDirectory = null; closeLoopClosurePair(); @@ -488,11 +591,18 @@ __RECONSTRUCTION_SCRIPT__ trackedKeypointOffsets = null; const empty = new Float32Array(); replacePointGeometry(empty, null); + installPointIds([]); + updateSelectedFrameObservations(0); replaceEstimatedGeometry(empty, empty); replaceLoopClosureGeometry(new Uint32Array(), new Float32Array()); error.style.display = "none"; error.textContent = ""; pauseKeyframePlayback(); + ++previewGeneration; + framePreviewCards = []; + document.getElementById("keyframe-previews").replaceChildren(); + keyframeTimeline.frames = []; + keyframeTimeline.frameImageOffsets = [0]; keyframeTimeline.names = []; keyframeTimeline.imageIds = []; keyframeTimeline.timestampsSeconds = null; @@ -501,14 +611,11 @@ __RECONSTRUCTION_SCRIPT__ slider.max = "0"; slider.value = "0"; slider.disabled = true; + document.getElementById("keyframe-previous").disabled = true; + document.getElementById("keyframe-next").disabled = true; document.getElementById("keyframe-play").disabled = true; document.getElementById("keyframe-playback-status").textContent = "timestamps unavailable"; document.getElementById("keyframe-label").textContent = ""; - const preview = document.getElementById("keyframe-preview"); - preview.removeAttribute("src"); - preview.removeAttribute("data-source-width"); - preview.removeAttribute("data-source-height"); - drawTrackedKeypoints(0); } function setCovarianceRankLimit(limit) { @@ -599,7 +706,8 @@ __RECONSTRUCTION_SCRIPT__ let visibleLoopClosureIndices = initialLoopClosures.edgeIndices; scene.add(loopClosures); function updateLoopClosureGeometry(maximumKeyframe) { - const visible = visibleLoopClosureData(maximumKeyframe); + const maximumImage = (keyframeTimeline.frameImageOffsets[maximumKeyframe + 1] ?? 0) - 1; + const visible = visibleLoopClosureData(maximumImage); visibleLoopClosurePositions = visible.positions; visibleLoopClosureIndices = visible.edgeIndices; const minimumInput = document.getElementById("loop-closures-min-shared"); @@ -713,13 +821,15 @@ __RECONSTRUCTION_SCRIPT__ updateLoopClosureGeometry(Number(document.getElementById("keyframe-slider").value)); fitScene(); } - function showKeyframe(index) { + async function showKeyframe(index) { if (keyframeTimeline.names.length === 0) return; - const bounded = Math.max(0, Math.min(index, keyframeTimeline.names.length - 1)); + const bounded = Math.max(0, Math.min(index, keyframeTimeline.frames.length - 1)); document.getElementById("keyframe-slider").value = String(bounded); - const pointCount = keyframeTimeline.pointOffsets[bounded + 1]; + if (!(await showFramePreviews(bounded))) return; + const imageCount = keyframeTimeline.frameImageOffsets[bounded + 1]; + const pointCount = keyframeTimeline.pointOffsets[imageCount]; if (covarianceIndexActive) { - keyframeLimit.value = bounded; + keyframeLimit.value = imageCount - 1; pointsGeometry.setDrawRange(0, covarianceDrawCount); } else { pointsGeometry.setDrawRange(0, pointCount); @@ -728,26 +838,19 @@ __RECONSTRUCTION_SCRIPT__ ? "" : ` Β· ${keyframeTimeline.timestampsSeconds[bounded].toFixed(3)} s`; document.getElementById("keyframe-label").textContent = - `${bounded + 1}/${keyframeTimeline.names.length}${timestampText} Β· ${keyframeTimeline.names[bounded]} Β· ${pointCount.toLocaleString()} cumulative points`; - const preview = document.getElementById("keyframe-preview"); - if (imageDirectory !== null) { - loadImage(preview, imageDirectory, keyframeTimeline.names[bounded]); - } - preview.dataset.keyframeIndex = String(bounded); - drawTrackedKeypoints(bounded); - updatePathPrefix(estimatedPath, estimatedPathPositions, bounded + 1); + `${bounded + 1}/${keyframeTimeline.frames.length}${timestampText} Β· frame ${keyframeTimeline.frames[bounded].frameId ?? keyframeTimeline.names[bounded]} Β· ${pointCount.toLocaleString()} cumulative points`; + updateSelectedFrameObservations(bounded); + updateFramePath(bounded); const estimatedSize = Number(document.getElementById("estimated-frusta-size").value); updateFrustaPrefix( - estimatedFrusta, estimatedFrustaPositions, estimatedPathPositions, bounded + 1, estimatedSize + estimatedFrusta, estimatedFrustaPositions, estimatedPathPositions, imageCount, estimatedSize ); updateLoopClosureGeometry(bounded); } - const keyframePreview = document.getElementById("keyframe-preview"); - keyframePreview.onload = () => drawTrackedKeypoints(Number(keyframePreview.dataset.keyframeIndex)); const keyframePreviewPanel = document.getElementById("keyframe-preview-panel"); keyframePreviewPanel.ontoggle = () => { if (keyframePreviewPanel.open) { - requestAnimationFrame(() => drawTrackedKeypoints(Number(keyframeSlider.value))); + requestAnimationFrame(() => drawTrackedKeypoints()); } }; const loopClosurePairClose = document.getElementById("loop-closure-pair-close"); @@ -755,7 +858,7 @@ __RECONSTRUCTION_SCRIPT__ if (typeof ResizeObserver !== "undefined") { const imagePanelResizeObserver = new ResizeObserver(() => { requestAnimationFrame(() => { - drawTrackedKeypoints(Number(keyframeSlider.value)); + drawTrackedKeypoints(); if (activeLoopClosurePair !== null) redrawLoopClosurePair(activeLoopClosurePair); }); }); @@ -839,7 +942,7 @@ __RECONSTRUCTION_SCRIPT__ function startKeyframePlayback() { if (keyframeTimeline.timestampsSeconds === null) return; let startIndex = Number(keyframeSlider.value); - if (startIndex >= keyframeTimeline.names.length - 1) { + if (startIndex >= keyframeTimeline.frames.length - 1) { startIndex = 0; showKeyframe(startIndex); } @@ -848,7 +951,8 @@ __RECONSTRUCTION_SCRIPT__ let displayedIndex = startIndex; keyframePlay.textContent = "pause"; keyframePlaybackStatus.textContent = `${speed}Γ— real time`; - function advance(now) { + async function advance(now) { + const request = keyframePlaybackRequest; const index = timelineIndexAtElapsed( keyframeTimeline.timestampsSeconds, startIndex, @@ -856,10 +960,11 @@ __RECONSTRUCTION_SCRIPT__ speed ); if (index !== displayedIndex) { - showKeyframe(index); + await showKeyframe(index); + if (request !== keyframePlaybackRequest) return; displayedIndex = index; } - if (index >= keyframeTimeline.names.length - 1) { + if (index >= keyframeTimeline.frames.length - 1) { pauseKeyframePlayback(); keyframePlaybackStatus.textContent = "finished"; return; @@ -872,6 +977,12 @@ __RECONSTRUCTION_SCRIPT__ pauseKeyframePlayback(); showKeyframe(Number(event.target.value)); }; + for (const [id, step] of [["keyframe-previous", -1], ["keyframe-next", 1]]) { + document.getElementById(id).onclick = () => { + pauseKeyframePlayback(); + showKeyframe(Number(keyframeSlider.value) + step); + }; + } keyframePlay.onclick = () => { if (keyframePlaybackRequest === null) startKeyframePlayback(); else pauseKeyframePlayback(); @@ -882,8 +993,12 @@ __RECONSTRUCTION_SCRIPT__ startKeyframePlayback(); } }; + document.getElementById("frame-observations-toggle").onchange = () => { + updateSelectedFrameObservations(Number(document.getElementById("keyframe-previews").dataset.frameIndex)); + invalidateSceneRender(); + }; const trackedKeypointsToggle = document.getElementById("tracked-keypoints-toggle"); - trackedKeypointsToggle.onchange = () => drawTrackedKeypoints(Number(keyframeSlider.value)); + trackedKeypointsToggle.onchange = () => drawTrackedKeypoints(); const estimatedFrustaToggle = document.getElementById("estimated-frusta-toggle"); estimatedFrustaToggle.onchange = event => estimatedFrusta.visible = event.target.checked; document.getElementById("paths-toggle").onchange = () => showKeyframe(Number(keyframeSlider.value)); @@ -1013,7 +1128,7 @@ __RECONSTRUCTION_SCRIPT__ for (const object of [estimatedFrusta, loopClosures, estimatedPath]) { object.material.resolution.set(window.innerWidth, window.innerHeight); } - drawTrackedKeypoints(Number(keyframeSlider.value)); + drawTrackedKeypoints(); if (activeLoopClosurePair !== null) redrawLoopClosurePair(activeLoopClosurePair); invalidateSceneRender(); } diff --git a/vidmap/visualization/html/browser/browser_worker.js b/vidmap/visualization/html/browser/browser_worker.js index dbbab02..141d816 100644 --- a/vidmap/visualization/html/browser/browser_worker.js +++ b/vidmap/visualization/html/browser/browser_worker.js @@ -60,18 +60,27 @@ const height = checkedCount(view.getBigUint64(offset + 16, true), "camera height"); offset += 24; requireBytes(view, offset, parameterCount * 8, "camera parameters"); - const fx = view.getFloat64(offset, true); - const fy = SINGLE_FOCAL_MODELS.has(modelId) ? fx : view.getFloat64(offset + 8, true); - const principalOffset = SINGLE_FOCAL_MODELS.has(modelId) ? 8 : 16; - const cx = view.getFloat64(offset + principalOffset, true); - const cy = view.getFloat64(offset + principalOffset + 8, true); - cameras.set(cameraId, {fx: fx, fy: fy, cx: cx, cy: cy, width: width, height: height}); + cameras.set(cameraId, cameraParameters(modelId, width, height, + new DataView(buffer, offset, parameterCount * 8))); offset += parameterCount * 8; } if (offset !== view.byteLength) throw new Error(`Unexpected trailing cameras.bin bytes: ${view.byteLength - offset}`); return cameras; } + function cameraParameters(modelId, width, height, view) { + const count = CAMERA_PARAMETER_COUNTS[modelId]; + if (count === undefined || view.byteLength !== count * 8) { + throw new Error("Invalid native camera parameters"); + } + const fx = view.getFloat64(0, true); + const fy = SINGLE_FOCAL_MODELS.has(modelId) ? fx : view.getFloat64(8, true); + const principalOffset = SINGLE_FOCAL_MODELS.has(modelId) ? 8 : 16; + const cx = view.getFloat64(principalOffset, true); + const cy = view.getFloat64(principalOffset + 8, true); + return {modelId: modelId, fx: fx, fy: fy, cx: cx, cy: cy, width: width, height: height}; + } + function quaternionRotation(w, x, y, z) { const norm = Math.hypot(w, x, y, z); if (!(norm > 0)) throw new Error("Invalid zero-norm image quaternion"); @@ -159,6 +168,17 @@ return candidates.length === 1 ? candidates[0] : null; } + function filenameTimestamps(names) { + const tokens = names.map(numericTimestampToken); + if (tokens.length === 0 || tokens.some(token => token === null)) return null; + const values = tokens.map(Number); + const unique = values.every( + (value, index) => Number.isFinite(value) && (index === 0 || value > values[index - 1]) + ); + const scale = unique ? inferredTimestampScale(tokens, values) : null; + return scale === null ? null : values.map(value => (value - values[0]) / scale); + } + function imageTimeline(images) { const entries = Array.from(images.entries()).map(([imageId, image]) => ({ imageId: imageId, @@ -174,16 +194,7 @@ } else { entries.sort((left, right) => left.imageId - right.imageId); } - let timestampsSeconds = null; - if (hasNumericNames && entries.length > 0) { - const tokens = entries.map(entry => entry.token); - const values = tokens.map(Number); - const unique = values.every( - (value, index) => Number.isFinite(value) && (index === 0 || value > values[index - 1]) - ); - const scale = unique ? inferredTimestampScale(tokens, values) : null; - if (scale !== null) timestampsSeconds = values.map(value => (value - values[0]) / scale); - } + const timestampsSeconds = filenameTimestamps(entries.map(entry => entry.name)); if (timestampsSeconds === null) entries.sort((left, right) => left.imageId - right.imageId); return { imageIds: entries.map(entry => entry.imageId), @@ -192,6 +203,105 @@ }; } + function parseFrames(buffer, images) { + const view = new DataView(buffer); + requireBytes(view, 0, 8, "frames.bin header"); + const count = checkedCount(view.getBigUint64(0, true), "frame count"); + const frames = new Map(); + const assigned = new Set(); + let offset = 8; + for (let index = 0; index < count; ++index) { + requireBytes(view, offset, 68, "frame record"); + const frameId = view.getUint32(offset, true); + const rigId = view.getUint32(offset + 4, true); + const dataCount = view.getUint32(offset + 64, true); + offset += 68; + requireBytes(view, offset, dataCount * 16, "frame data IDs"); + if (frames.has(frameId)) throw new Error("Duplicate COLMAP frame ID"); + const members = new Map(); + for (let item = 0; item < dataCount; ++item, offset += 16) { + const type = view.getInt32(offset, true); + const cameraId = view.getUint32(offset + 4, true); + const imageId = checkedCount(view.getBigUint64(offset + 8, true), "frame image ID"); + if (type !== 0) continue; + if (members.has(cameraId) || assigned.has(imageId) || + (images.has(imageId) && images.get(imageId).cameraId !== cameraId)) { + throw new Error("COLMAP frame membership contradicts images.bin"); + } + members.set(cameraId, imageId); + if (images.has(imageId)) assigned.add(imageId); + } + frames.set(frameId, {frameId, rigId, members}); + } + if (offset !== view.byteLength || assigned.size !== images.size) { + throw new Error("COLMAP frames must cover all registered images exactly once"); + } + return frames; + } + + function frameTimeline(images, framesBuffer = null, sequence = null) { + const actual = framesBuffer === null ? null : parseFrames(framesBuffer, images); + const camerasByRig = new Map(); + for (const frame of actual?.values() ?? []) { + if (!camerasByRig.has(frame.rigId)) camerasByRig.set(frame.rigId, new Set()); + for (const cameraId of frame.members.keys()) camerasByRig.get(frame.rigId).add(cameraId); + } + if (sequence === null && Array.from(camerasByRig.values()).every(cameras => cameras.size <= 1)) { + const ordinary = imageTimeline(images); + return {...ordinary, + frames: ordinary.imageIds.map((id, i) => ({frameId: null, rigId: null, + cameraIds: [images.get(id).cameraId], imageIndices: [i]})), + frameImageOffsets: ordinary.imageIds.map((_, i) => i).concat([ordinary.imageIds.length])}; + } + if (actual === null) throw new Error("Capture grouping requires actual COLMAP frames.bin"); + const timesByFrame = new Map(); + let ordered; + if (sequence !== null) { + const ids = sequence.frame_ids, times = sequence.timestamps_ns; + if (Object.keys(sequence).sort().join(',') !== 'frame_ids,timestamps_ns' || + !Array.isArray(ids) || ids.length === 0 || new Set(ids).size !== ids.length || + ids.some(id => !Number.isSafeInteger(id) || id < 0 || id >= 4294967295) || + (times !== null && (!Array.isArray(times) || times.length !== ids.length || + times.some((time, i) => typeof time !== 'bigint' || (i > 0 && time <= times[i-1]))))) { + throw new Error("Invalid mapper frame sequence"); + } + const selected = new Set(ids); + if (Array.from(actual.keys()).some(id => !selected.has(id))) { + throw new Error("Mapper frame sequence does not cover COLMAP frames"); + } + if (times !== null) ids.forEach((id, i) => timesByFrame.set(id, times[i])); + ordered = ids.filter(id => actual.has(id)).map(id => actual.get(id)); + } else { + ordered = Array.from(actual.values()).sort((a,b) => a.frameId-b.frameId); + } + for (const [rigId, cameras] of camerasByRig) { + camerasByRig.set(rigId, Array.from(cameras).sort((a,b) => a-b)); + } + const imageIds = [], names = [], frames = [], frameImageOffsets = [0], times = []; + for (const record of ordered) { + if (!Array.from(record.members.values()).some(id => images.has(id))) continue; + const cameraIds = camerasByRig.get(record.rigId); + const imageIndices = cameraIds.map(cameraId => { + const id = record.members.get(cameraId); + if (!images.has(id)) return null; + const index = imageIds.length; + imageIds.push(id); + names.push(images.get(id).name); + return index; + }); + frames.push({frameId:record.frameId,rigId:record.rigId,cameraIds,imageIndices}); + frameImageOffsets.push(imageIds.length); + if (timesByFrame.size) times.push(timesByFrame.get(record.frameId)); + } + if (frames.length === 0) throw new Error("No registered capture frames"); + // Native grouping/order stays authoritative. Ordinary single-image captures + // may omit explicit timing even when filenames already encode it. + const inferred = frames.every(frame => frame.cameraIds.length === 1) + ? filenameTimestamps(names) : null; + return {imageIds,names,frames,frameImageOffsets, + timestampsSeconds:times.length === 0 ? inferred : times.map(value => Number(value-times[0])/1e9)}; + } + function cameraCenter(image) { const r = image.rotation; const t = image.translation; @@ -481,6 +591,8 @@ parseCameras: parseCameras, parseImages: parseImages, imageTimeline: imageTimeline, + frameTimeline: frameTimeline, + parseFrames: parseFrames, cameraGeometry: cameraGeometry, parsePointsForDisplay: parsePointsForDisplay, computeCovariances: computeCovariances, @@ -495,7 +607,7 @@ if (message.command === "load") { const buffers = message.buffers; const images = parseImages(buffers["images.bin"]); - const timeline = imageTimeline(images); + const timeline = frameTimeline(images, message.framesBuffer ?? null, message.frameSequence ?? null); const keyframeImageIds = timeline.imageIds; const points = parsePointsForDisplay( buffers["points3D.bin"], diff --git a/vidmap/visualization/html/browser/vidmap-viewer.html.in b/vidmap/visualization/html/browser/vidmap-viewer.html.in index 13d4dd1..f4f96ca 100644 --- a/vidmap/visualization/html/browser/vidmap-viewer.html.in +++ b/vidmap/visualization/html/browser/vidmap-viewer.html.in @@ -66,12 +66,20 @@ #keyframe-preview-panel summary { width: max-content; cursor: pointer; font-weight: 600; user-select: none; } - #keyframe-preview { + #keyframe-previews { position: relative; display: flex; gap: 8px; align-items: flex-start; } + #keyframe-preview-panel.rig-frame[open] { width: min(88vw, 1500px); } + .keyframe-camera { flex: 1; min-width: 0; margin: 0; } + .keyframe-camera figcaption { overflow-wrap: anywhere; font-size: 11px; } + #keyframe-previews[data-loading="true"]::after { + content: "Loading synchronized images…"; position: absolute; top: 0; right: 0; + padding: 2px 4px; border-radius: 3px; background: rgba(255,255,255,.9); + } + .keyframe-preview { display: block; width: 100%; height: auto; max-height: 70vh; object-fit: contain; border-radius: 3px; background: #111; } - #keyframe-preview-stage { position: relative; width: 100%; margin-top: 4px; line-height: 0; } - #tracked-keypoints-overlay { + .keyframe-preview-stage { position: relative; width: 100%; margin-top: 4px; line-height: 0; } + .tracked-keypoints-overlay { position: absolute; inset: 0; width: 100%; height: 100%; pointer-events: none; } #loop-closure-pair-panel { @@ -113,11 +121,13 @@ __SQLITE_SCRIPT__
Keyframe -
- Selected keyframe - -
+
+