From 5ec565673e8be7fca2f6a84c8cc668e9ee0678a1 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Sun, 18 Oct 2026 17:00:42 -0500 Subject: [PATCH 01/11] fix(push): keep local sync running while a watch push runs Local session ingestion must keep running during remote pushes. Push cutoffs and rebuild recovery must also preserve every change while the archive is being written or replaced. - fix(push): take each push cutoff under the archive write lock - fix(push): retry a failed startup sync on the debounce - fix(postgres): take the push cutoff before listing scope moves - refactor(push): run CLI and daemon pushes through one shared path - refactor(push): keep each watch push rule in one place - fix(push): let one daemon push run at a time and keep --full on retry - fix(push): close gaps that let a push skip or keep stale rows - fix(push): keep the final sync and --full through daemon failures - fix(push): handle archive rebuilds and writer handoffs during a push - fix(push): push in full after a rebuild and tell when a daemon copy started - test(clickhouse): fail the full push for real after it records its run - fix(push): keep --full only when needed and explain daemon push waits - fix(postgres): remove obsolete sessions after archive rebuilds Co-authored-by: Claude Opus 5.5 Co-authored-by: Wes McKinney --- cmd/agentsview/archive_push_decouple_test.go | 668 ++++++++++++++++ cmd/agentsview/archive_sync_lane.go | 391 +++++++++ cmd/agentsview/archive_write_backend.go | 742 +++++++----------- cmd/agentsview/archive_write_backend_test.go | 323 +++++--- cmd/agentsview/daemon_push.go | 49 +- cmd/agentsview/daemon_push_test.go | 75 ++ cmd/agentsview/duckdb.go | 55 +- cmd/agentsview/pg_watch_loop.go | 23 +- cmd/agentsview/pg_watch_test.go | 193 +++-- cmd/agentsview/replica_embed_test.go | 60 +- cmd/agentsview/replica_watch.go | 216 +++-- docs/pg-sync.md | 8 + internal/archivepush/archivepush.go | 146 ++++ internal/clickhouse/push.go | 19 +- .../push_not_started_chtest_test.go | 42 + internal/clickhouse/sync.go | 8 +- internal/db/db.go | 32 + internal/db/db_test.go | 13 + internal/db/orphaned.go | 2 + internal/db/sessions.go | 77 +- internal/db/sessions_sync_marker_test.go | 192 +++++ internal/duckdb/rebuild.go | 26 +- internal/duckdb/rebuild_test.go | 20 + internal/duckdb/sync.go | 34 +- internal/duckdb/sync_fastpath_test.go | 26 + internal/postgres/handoff_sync_state_test.go | 76 ++ internal/postgres/push.go | 247 ++++-- .../postgres/push_generation_pgtest_test.go | 202 +++++ internal/postgres/push_pgtest_test.go | 112 ++- internal/postgres/sync.go | 104 ++- internal/server/huma_routes_push.go | 353 ++++++--- .../server/huma_routes_push_internal_test.go | 352 ++++++++- internal/server/server.go | 2 + internal/storage/mirror.go | 9 + internal/storage/push.go | 26 + internal/sync/engine.go | 33 + internal/sync/progress.go | 6 + internal/sync/signal_schedule_test.go | 55 ++ internal/testutil/goroutine.go | 22 + 39 files changed, 3988 insertions(+), 1051 deletions(-) create mode 100644 cmd/agentsview/archive_push_decouple_test.go create mode 100644 cmd/agentsview/archive_sync_lane.go create mode 100644 internal/archivepush/archivepush.go create mode 100644 internal/clickhouse/push_not_started_chtest_test.go create mode 100644 internal/postgres/handoff_sync_state_test.go create mode 100644 internal/postgres/push_generation_pgtest_test.go create mode 100644 internal/testutil/goroutine.go diff --git a/cmd/agentsview/archive_push_decouple_test.go b/cmd/agentsview/archive_push_decouple_test.go new file mode 100644 index 0000000000..70a79d384d --- /dev/null +++ b/cmd/agentsview/archive_push_decouple_test.go @@ -0,0 +1,668 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/archivepush" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/dbtest" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/storage" + syncpkg "go.kenn.io/agentsview/internal/sync" + "go.kenn.io/agentsview/internal/testjsonl" + "go.kenn.io/agentsview/internal/testutil" +) + +// blockedRemotePush stands in for an hour-long full push: it reports that it +// started and holds until the test releases it or its context ends. +type blockedRemotePush struct { + started chan struct{} + release chan struct{} +} + +func newBlockedRemotePush() *blockedRemotePush { + return &blockedRemotePush{ + started: make(chan struct{}, 8), + release: make(chan struct{}), + } +} + +func (b *blockedRemotePush) wait(ctx context.Context) error { + select { + case b.started <- struct{}{}: + default: + } + select { + case <-b.release: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (*blockedRemotePush) EnsureSchema(context.Context) error { return nil } +func (b *blockedRemotePush) PushWithOptions( + ctx context.Context, _ storage.PushOptions, _ func(storage.PushProgress), +) (storage.PushResult, error) { + return storage.PushResult{}, b.wait(ctx) +} +func (*blockedRemotePush) Close() error { return nil } + +// A remote push that never finishes must not stop the local archive from +// ingesting a session file written while the push runs. +func TestLocalPushWatchIngestsWhileRemotePushBlocks(t *testing.T) { + owners := []struct { + name string + run func(context.Context, *localArchiveWriteBackend, *blockedRemotePush) error + }{ + { + name: "replica", + run: func( + ctx context.Context, b *localArchiveWriteBackend, remote *blockedRemotePush, + ) error { + b.watchHooks = &archivePushWatchHooks{ + newReplicaPusher: func(*syncpkg.Engine) *replicaPusher { + return &replicaPusher{ + label: "clickhouse watch", displayName: "ClickHouse", + session: archivepush.ReplicaSession{ + Connect: func(context.Context) (storage.Pusher, error) { + return remote, nil + }, + }, + vectors: newVectorReconcile(), + } + }, + } + return b.ReplicaPushWatch( + ctx, pgReplica{}, storage.ConfiguredReplica{}, ReplicaPushConfig{}, + nil, nil, time.Hour, time.Hour, + ) + }, + }, + { + name: "duckdb", + run: func( + ctx context.Context, b *localArchiveWriteBackend, remote *blockedRemotePush, + ) error { + b.watchHooks = &archivePushWatchHooks{ + newDuckDBPusher: func(*syncpkg.Engine) *duckDBPusher { + return &duckDBPusher{ + mirrorPush: func( + c context.Context, _ bool, + ) (storage.MirrorPushResult, error) { + return storage.MirrorPushResult{}, remote.wait(c) + }, + } + }, + } + return b.DuckDBPushWatch( + ctx, config.DuckDBConfig{}, DuckDBPushConfig{}, nil, nil, + time.Hour, time.Hour, + ) + }, + }, + } + for _, owner := range owners { + t.Run(owner.name, func(t *testing.T) { + dataDir := t.TempDir() + dbPath := filepath.Join(dataDir, "sessions.db") + database := dbtest.OpenTestDBAt(t, dbPath) + codexRoot := t.TempDir() + backend := &localArchiveWriteBackend{ + appCfg: config.Config{ + DataDir: dataDir, + DBPath: dbPath, + InstallationID: "local", + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {codexRoot}, + }, + }, + database: database, + } + remote := newBlockedRemotePush() + + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + done := make(chan error, 1) + go func() { done <- owner.run(ctx, backend, remote) }() + + select { + case <-remote.started: + case err := <-done: + require.FailNowf(t, "watch exited before the startup push", "%v", err) + case <-time.After(10 * time.Second): + require.FailNow(t, "startup push never began") + } + + uuid := "a1b2c3d4-1111-4222-8333-444455556666" + day := filepath.Join(codexRoot, "2026", "10", "15") + require.NoError(t, os.MkdirAll(day, 0o755)) + content := testjsonl.NewSessionBuilder(). + AddCodexMeta( + "2026-10-15T14:00:00Z", uuid, "/home/user/code/api", + "codex_cli_rs", + ). + AddCodexMessage("2026-10-15T14:00:01Z", "user", "hello"). + String() + require.NoError(t, os.WriteFile( + filepath.Join(day, "rollout-2026-10-15T14-00-00-"+uuid+".jsonl"), + []byte(content), 0o644, + )) + + require.Eventually(t, func() bool { + session, err := database.GetSession(t.Context(), "codex:"+uuid) + return err == nil && session != nil + }, 10*time.Second, 20*time.Millisecond, + "a session written during a blocked remote push must reach the archive") + + close(remote.release) + cancel() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(30 * time.Second): + require.FailNow(t, "watch did not shut down") + } + }) + } +} + +// Local owners open watcher dispatch as soon as startup sync finishes, so the +// startup push and its retries never hold back watcher batches. +func TestLocalPushWatchOpensDispatchBeforeStartupPush(t *testing.T) { + for _, owner := range localPushWatchOwnerCases(t) { + t.Run(owner.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + h := newPushWatchOwnerHarness(1) + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + done := make(chan error, 1) + go func() { done <- owner.run(ctx, h.hooks()) }() + + receiveArchiveTest(t, h.opened) + first := receiveArchiveTest(t, h.attempts) + require.Equal(t, 1, first.index) + synctest.Wait() + events, opens, _ := h.snapshot() + assert.Equal(t, 1, opens) + assert.Less(t, indexOfEvent(events, "startup-sync"), + indexOfEvent(events, "open")) + assert.Less(t, indexOfEvent(events, "open"), + indexOfEvent(events, "push")) + pending, waiters := h.pending() + require.True(t, pending && waiters == 0, + "a partial startup push stays queued for retry") + + // An authoritative batch syncs locally and returns without + // waiting for the push it queues. + require.NoError(t, h.watcherCallback()( + ctx, syncpkg.WatchBatch{FullSync: true}, + )) + + h.floor <- time.Now() + second := receiveArchiveTest(t, h.attempts) + require.Equal(t, 2, second.index) + synctest.Wait() + pending, _ = h.pending() + assert.False(t, pending, "the retry covers the queued batch") + + cancel() + require.NoError(t, receiveArchiveTest(t, done)) + }) + }) + } +} + +// When startup sync fails, local owners keep dispatch closed, skip pushes, +// and retry the startup sync, with its --full rebuild, until one completes. +// The --full request survives the failure: the first push after that resends +// every session, and later pushes do not. +func TestLocalPushWatchHoldsPushesUntilLocalSyncCompletes(t *testing.T) { + for _, owner := range localPushWatchOwnerCases(t) { + t.Run(owner.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + h := newPushWatchOwnerHarness(1) + hooks := h.hooks() + var attempts atomic.Int32 + hooks.startupSync = func( + _ context.Context, _ *syncpkg.Engine, full bool, + ) (bool, error) { + h.record("startup-sync") + assert.True(t, full, "each startup sync attempt must get the --full request") + if attempts.Add(1) == 1 { + return false, errors.New("startup sync discovery incomplete") + } + return false, nil + } + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + done := make(chan error, 1) + go func() { done <- owner.runFull(ctx, hooks) }() + + synctest.Wait() + h.floor <- time.Now() + synctest.Wait() + _, opens, _ := h.snapshot() + require.Zero(t, opens, "dispatch stays closed until a local sync completes") + select { + case attempt := <-h.attempts: + require.Failf(t, "pushed before a complete local sync", + "attempt %d", attempt.index) + default: + } + + // The local sync lane retries on the debounce, well before the + // floor interval. + time.Sleep(localPushWatchTestDebounce) + synctest.Wait() + receiveArchiveTest(t, h.opened) + assert.Equal(t, int32(2), attempts.Load(), + "the retry reruns the startup sync, so a rebuild it needs still runs") + + h.fire <- time.Now() + first := receiveArchiveTest(t, h.attempts) + assert.True(t, first.full, "--full makes the first push full") + synctest.Wait() + + h.floor <- time.Now() + retry := receiveArchiveTest(t, h.attempts) + assert.False(t, retry.full, + "the mirror resumes a failed full push, so a retry must not start it over") + + cancel() + require.NoError(t, receiveArchiveTest(t, done)) + }) + }) + } +} + +// A startup rebuild can replace the archive and still report failures. The +// watch must then make its first push full, because only a full push removes +// remote rows that the rebuild dropped. A --full retry must not rebuild the +// archive a second time. +func TestLocalPushWatchPushesFullAfterStartupRebuildWithFailures(t *testing.T) { + for _, full := range []bool{false, true} { + for _, owner := range localPushWatchOwnerCases(t) { + t.Run(fmt.Sprintf("%s full=%t", owner.name, full), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + h := newPushWatchOwnerHarness() + hooks := h.hooks() + var fullArgs []bool + hooks.startupSync = func( + ctx context.Context, _ *syncpkg.Engine, fullArg bool, + ) (bool, error) { + fullArgs = append(fullArgs, fullArg) + if len(fullArgs) == 1 { + require.NoError(t, owner.database.SetDatabaseIDForTest(ctx, "rebuilt")) + return false, errors.New("startup sync processing incomplete") + } + return false, nil + } + run := owner.run + if full { + run = owner.runFull + } + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + done := make(chan error, 1) + go func() { done <- run(ctx, hooks) }() + + synctest.Wait() + time.Sleep(localPushWatchTestDebounce) + synctest.Wait() + receiveArchiveTest(t, h.opened) + h.fire <- time.Now() + first := receiveArchiveTest(t, h.attempts) + assert.True(t, first.full, "the first push after a rebuild must be full") + assert.Equal(t, []bool{full, false}, fullArgs, + "a retry after a rebuild must not rebuild again") + + cancel() + require.NoError(t, receiveArchiveTest(t, done)) + }) + }) + } + } +} + +// A retry of the startup sync can rebuild the whole archive, so repeated +// failures space the retries out: the wait doubles each time, up to the floor +// interval. +func TestLocalPushWatchBacksOffFailedStartupSyncs(t *testing.T) { + for _, owner := range localPushWatchOwnerCases(t) { + t.Run(owner.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + h := newPushWatchOwnerHarness() + hooks := h.hooks() + start := time.Now() + var attempts []time.Duration + hooks.startupSync = func(context.Context, *syncpkg.Engine, bool) (bool, error) { + attempts = append(attempts, time.Since(start)) + return false, errors.New("startup sync discovery incomplete") + } + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + done := make(chan error, 1) + go func() { done <- owner.run(ctx, hooks) }() + + time.Sleep(7*localPushWatchTestDebounce + time.Second) + synctest.Wait() + d := localPushWatchTestDebounce + assert.Equal(t, []time.Duration{0, d, 3 * d, 7 * d}, attempts) + + cancel() + require.NoError(t, receiveArchiveTest(t, done)) + }) + }) + } +} + +// A periodic local sync queues a push only when it writes the archive. +func TestArchiveSyncLaneQueuesPushOnlyAfterArchiveChanges(t *testing.T) { + database := dbtest.OpenTestDB(t) + codexRoot := t.TempDir() + engine := syncpkg.NewEngine(t.Context(), database, syncpkg.EngineConfig{ + AgentDirs: map[parser.AgentType][]string{parser.AgentCodex: {codexRoot}}, + }) + t.Cleanup(engine.Close) + lane := &archiveSyncLane{ + label: "test watch", engine: engine, openDispatch: func() {}, + } + lane.ready.Store(true) + newLoop := func() { + loop, ticker := newPushLoopWithLabel("test watch", time.Hour, time.Hour, + func(context.Context, pushReason, *syncpkg.WatchBatch) error { return nil }) + t.Cleanup(ticker.Stop) + lane.loop = loop + } + queued := func() bool { + pending, _ := pushLoopPendingState(lane.loop) + return pending + } + ctx := t.Context() + + newLoop() + // The first sync of a new archive records its baseline. + lane.syncAll(ctx) + + newLoop() + lane.syncAll(ctx) + assert.False(t, queued(), "a sync that wrote nothing queues no push") + + uuid := "b2c3d4e5-2222-4333-8444-555566667777" + day := filepath.Join(codexRoot, "2026", "10", "15") + require.NoError(t, os.MkdirAll(day, 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(day, "rollout-2026-10-15T15-00-00-"+uuid+".jsonl"), + []byte(testjsonl.NewSessionBuilder(). + AddCodexMeta("2026-10-15T15:00:00Z", uuid, "/home/user/code/api", "codex_cli_rs"). + AddCodexMessage("2026-10-15T15:00:01Z", "user", "hello"). + String()), + 0o644, + )) + lane.syncAll(ctx) + assert.True(t, queued(), "a sync that wrote a session queues a push") +} + +// Stopping a local watch archives changes the watcher had not dispatched yet, +// so the final push includes them. +func TestArchiveSyncLaneSyncsBeforeFinalPush(t *testing.T) { + database := dbtest.OpenTestDB(t) + codexRoot := t.TempDir() + engine := syncpkg.NewEngine(t.Context(), database, syncpkg.EngineConfig{ + AgentDirs: map[parser.AgentType][]string{parser.AgentCodex: {codexRoot}}, + }) + t.Cleanup(engine.Close) + lane := &archiveSyncLane{ + label: "test watch", engine: engine, openDispatch: func() {}, + } + lane.ready.Store(true) + uuid := "c3d4e5f6-3333-4444-8555-666677778888" + day := filepath.Join(codexRoot, "2026", "10", "16") + require.NoError(t, os.MkdirAll(day, 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(day, "rollout-2026-10-16T09-00-00-"+uuid+".jsonl"), + []byte(testjsonl.NewSessionBuilder(). + AddCodexMeta("2026-10-16T09:00:00Z", uuid, "/home/user/code/api", "codex_cli_rs"). + AddCodexMessage("2026-10-16T09:00:01Z", "user", "hello"). + String()), + 0o644, + )) + + var archived int + require.NoError(t, lane.push(t.Context(), reasonShutdown, nil, func( + ctx context.Context, _ pushReason, _ bool, _ *syncpkg.WatchBatch, + ) error { + var err error + archived, err = database.CountSessionsForMirrorScope(ctx, nil, nil) + return err + })) + assert.Equal(t, 1, archived, "the final push must see the pending session") +} + +// Stopping a local watch during its first push skips the final push but +// still archives changes the watcher had not handed over. +func TestLocalPushWatchSyncsWhenStoppedDuringStartupPush(t *testing.T) { + backend := testLocalArchiveWriteBackend(t) + codexRoot := t.TempDir() + backend.appCfg.AgentDirs = map[parser.AgentType][]string{parser.AgentCodex: {codexRoot}} + h := newPushWatchOwnerHarness() + hooks := h.hooks() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var pushes atomic.Int32 + hooks.newDuckDBPusher = func(*syncpkg.Engine) *duckDBPusher { + return &duckDBPusher{mirrorPush: func( + pctx context.Context, _ bool, + ) (storage.MirrorPushResult, error) { + if pushes.Add(1) > 1 { + assert.Fail(t, "stopping during the first push must not push again") + return storage.MirrorPushResult{}, nil + } + // A session changes while the first push runs, and the + // watcher has not handed it over when the watch stops. + uuid := "d4e5f6a7-4444-4555-8666-777788889999" + day := filepath.Join(codexRoot, "2026", "10", "17") + require.NoError(t, os.MkdirAll(day, 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(day, "rollout-2026-10-17T09-00-00-"+uuid+".jsonl"), + []byte(testjsonl.NewSessionBuilder(). + AddCodexMeta("2026-10-17T09:00:00Z", uuid, "/home/user/code/api", "codex_cli_rs"). + AddCodexMessage("2026-10-17T09:00:01Z", "user", "hello"). + String()), + 0o644, + )) + cancel() + <-pctx.Done() + return storage.MirrorPushResult{}, pctx.Err() + }} + } + backend.watchHooks = hooks + + require.NoError(t, backend.DuckDBPushWatch( + ctx, config.DuckDBConfig{}, DuckDBPushConfig{}, nil, nil, time.Hour, time.Hour, + )) + archived, err := backend.database.CountSessionsForMirrorScope(t.Context(), nil, nil) + require.NoError(t, err) + assert.Equal(t, 1, archived, "the stopped watch must archive the pending session") +} + +// A push waits for an in-progress local sync, so it never copies a session +// that the sync has only half written. +func TestArchiveSyncLaneSettlesBeforePush(t *testing.T) { + database := dbtest.OpenTestDB(t) + engine := syncpkg.NewEngine(t.Context(), database, syncpkg.EngineConfig{}) + t.Cleanup(engine.Close) + lane := &archiveSyncLane{ + label: "test watch", engine: engine, openDispatch: func() {}, + } + lane.ready.Store(true) + + held := make(chan struct{}) + release := make(chan struct{}) + releaseSync := sync.OnceFunc(func() { close(release) }) + // Runs before the engine closes, which waits for the held sync. + t.Cleanup(releaseSync) + syncDone := make(chan error, 1) + go func() { + syncDone <- engine.RunExclusive(func() error { + close(held) + <-release + return nil + }) + }() + <-held + + var pushed atomic.Bool + pushDone := make(chan error, 1) + go func() { + pushDone <- lane.push(t.Context(), reasonChange, nil, func( + context.Context, pushReason, bool, *syncpkg.WatchBatch, + ) error { + pushed.Store(true) + return nil + }) + }() + require.Eventually(t, func() bool { + return pushed.Load() || testutil.GoroutineBlockedOnMutexIn("(*Engine).SettleForPush") + }, 10*time.Second, 5*time.Millisecond) + require.False(t, pushed.Load(), "a push must wait for an in-progress sync") + + releaseSync() + require.NoError(t, <-syncDone) + require.NoError(t, <-pushDone) + assert.True(t, pushed.Load()) +} + +// A --full push that fails before it reaches the destination recorded no +// full push to resume, so the next push stays full. A push that fails after +// it started resumes on its own, so asking for full again would restart it. +func TestArchiveSyncLaneKeepsFullUntilAPushReachesTheDestination(t *testing.T) { + lane := &archiveSyncLane{label: "test watch", openDispatch: func() {}} + lane.fullPending.Store(true) + var fullArgs []bool + push := func(results ...error) { + for _, result := range results { + err := lane.push(t.Context(), reasonChange, nil, func( + _ context.Context, _ pushReason, full bool, _ *syncpkg.WatchBatch, + ) error { + fullArgs = append(fullArgs, full) + return result + }) + require.ErrorIs(t, err, result) + } + } + unreachable := errors.New("daemon unreachable") + copyFailed := errors.New("copy failed") + push(storage.PushNotStartedError{Err: unreachable}, copyFailed, copyFailed) + assert.Equal(t, []bool{true, true, false}, fullArgs) +} + +// Shutdown stops the watcher before the final push. A watcher batch can hold +// the sync lock until the watcher cancels it, and the final push waits for +// that lock, so stopping the watcher afterwards would stall shutdown. +func TestLocalPushWatchStopsWatcherBeforeFinalPush(t *testing.T) { + backend := testLocalArchiveWriteBackend(t) + var mu sync.Mutex + var events []string + record := func(event string) { + mu.Lock() + defer mu.Unlock() + events = append(events, event) + } + callbackCtx, stopCallbacks := context.WithCancel(t.Context()) + t.Cleanup(stopCallbacks) + engines := make(chan *syncpkg.Engine, 1) + pushed := make(chan struct{}, 2) + idle := make(chan struct{}, 1) + target := &fakeTarget{onPush: func() { + record("push") + pushed <- struct{}{} + }} + backend.watchHooks = &archivePushWatchHooks{ + startWatcher: func( + config.Config, *syncpkg.Engine, syncpkg.WatchCallback, + syncpkg.WatcherOptions, + ) (func(), func(), []string) { + return func() { + record("watcher stopped") + stopCallbacks() + }, func() {}, nil + }, + startupSync: func( + _ context.Context, engine *syncpkg.Engine, _ bool, + ) (bool, error) { + engines <- engine + return false, nil + }, + newReplicaPusher: func(*syncpkg.Engine) *replicaPusher { + return &replicaPusher{ + session: archivepush.ReplicaSession{ + Connect: func(context.Context) (storage.Pusher, error) { + return target, nil + }, + }, + vectors: newVectorReconcile(), + } + }, + newLoop: func( + label string, debounce, interval time.Duration, + push func(context.Context, pushReason, *syncpkg.WatchBatch) error, + ) (*pushLoop, func()) { + loop, ticker := newPushLoopWithLabel(label, debounce, interval, push) + loop.onIdle = func() { + select { + case idle <- struct{}{}: + default: + } + } + return loop, ticker.Stop + }, + } + + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + done := make(chan error, 1) + go func() { + done <- backend.ReplicaPushWatch( + ctx, pgReplica{}, storage.ConfiguredReplica{}, ReplicaPushConfig{}, + nil, nil, time.Hour, time.Hour, + ) + }() + engine := <-engines + <-pushed + // Cancelling during the startup push skips the final push, so wait for + // the loop to start waiting for work. + <-idle + + // Stand in for a watcher batch that syncs until the watcher stops it. + held := make(chan struct{}) + batchDone := make(chan error, 1) + go func() { + batchDone <- engine.RunExclusive(func() error { + close(held) + <-callbackCtx.Done() + return nil + }) + }() + <-held + + cancel() + require.NoError(t, <-done) + require.NoError(t, <-batchDone) + mu.Lock() + defer mu.Unlock() + assert.Equal(t, []string{"push", "watcher stopped", "push"}, events, + "the final push runs after the watcher stops") +} diff --git a/cmd/agentsview/archive_sync_lane.go b/cmd/agentsview/archive_sync_lane.go new file mode 100644 index 0000000000..bfc1d8cc26 --- /dev/null +++ b/cmd/agentsview/archive_sync_lane.go @@ -0,0 +1,391 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + stdsync "sync" + "sync/atomic" + "time" + + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/storage" + syncpkg "go.kenn.io/agentsview/internal/sync" +) + +// archivePushWatch runs a push-watch process. The same loop serves both +// archive owners. When this process owns the SQLite archive, engine is set: +// watcher batches and periodic syncs write the archive on their own +// goroutines, and each push copies whatever the archive holds when it starts, +// so a push that takes an hour never delays ingestion. When the daemon owns +// the archive, engine is nil: the daemon syncs before each push, and watcher +// batches travel with the push requests. +type archivePushWatch struct { + hooks *archivePushWatchHooks + appCfg config.Config + engine *syncpkg.Engine + // database is the archive engine writes. Set it with engine. + database *db.DB + label string + debounce, interval time.Duration + full bool + // push runs one push. batch is the watcher scope a daemon owner syncs + // before it pushes; a local owner has already synced it. + push func(ctx context.Context, reason pushReason, full bool, batch *syncpkg.WatchBatch) error + wake <-chan struct{} +} + +func (w archivePushWatch) run(ctx context.Context) error { + lane := &archiveSyncLane{ + label: w.label, + engine: w.engine, + retry: w.debounce, + maxRetry: w.interval, + requests: make(chan struct{}, 1), + } + lane.fullPending.Store(w.full) + loop, stopLoop := newArchivePushLoop( + w.hooks, w.label, w.debounce, w.interval, + func(c context.Context, r pushReason, batch *syncpkg.WatchBatch) error { + return lane.push(c, r, batch, w.push) + }, + ) + defer stopLoop() + lane.loop = loop + + callback := func(c context.Context, batch syncpkg.WatchBatch) error { + return notifyPushForWatchBatchWithConfig(c, loop, w.appCfg, batch) + } + options := syncpkg.WatcherOptions{OnCoverageDegraded: loop.NotifyCoverageDegraded} + unwatchedCoverage := fmt.Sprintf("relying on the %s floor for coverage", w.interval) + if w.engine != nil { + poller := newArchivePushUnwatchedPoller(ctx, w.hooks, w.engine) + defer poller.Stop() + callback = lane.watchCallback(w.appCfg) + options = archivePushWatchWatcherOptions(lane.notifyCoverageDegraded, poller) + unwatchedCoverage = fmt.Sprintf("polling every %s", unwatchedPollInterval) + } + stopWatcher, openDispatch, unwatchedDirs := startArchivePushWatcher( + w.hooks, w.appCfg, w.engine, callback, options, + ) + stopWatcher = stdsync.OnceFunc(stopWatcher) + defer stopWatcher() + // Stop the watcher as soon as shutdown starts. A watcher batch can hold + // the sync lock, and the final push waits for that lock. + defer context.AfterFunc(ctx, stopWatcher)() + lane.openDispatch = openDispatch + if len(unwatchedDirs) > 0 { + log.Printf( + "%s: %d root(s) not watched; %s", + w.label, len(unwatchedDirs), unwatchedCoverage, + ) + } + + wake := w.wake + if w.engine != nil { + lane.startupSync = runPGWatchStartupSync + if w.hooks != nil && w.hooks.startupSync != nil { + lane.startupSync = w.hooks.startupSync + } + lane.full = w.full + lane.database = w.database + startDatabaseID, err := w.database.GetDatabaseID(ctx) + if err != nil { + if ctx.Err() != nil { + return nil + } + return fmt.Errorf("reading archive database id: %w", err) + } + lane.startDatabaseID = startDatabaseID + lane.syncStartup(ctx) + if ctx.Err() != nil { + return nil + } + + ticker := time.NewTicker(w.interval) + defer ticker.Stop() + laneDone := make(chan struct{}) + go func() { + defer close(laneDone) + lane.run(ctx, ticker.C, w.wake) + }() + defer func() { <-laneDone }() + // The lane turns lifecycle wakes into syncs, and a sync that writes + // the archive queues a push. + wake = nil + } + + if lane.canPush() { + // The startup push runs on this goroutine before the loop starts, so + // it skips the change debounce. A failed startup push stays queued + // and retries on the debounce. + loop.enqueueUnscoped(reasonStartup, nil, false) + loop.runNextTask(ctx, false) + if ctx.Err() != nil { + // Stopping during the first push skips the final push, which would + // repeat the push just interrupted. A local owner still archives + // the changes the watcher had not handed over. + if w.engine != nil { + flushCtx, cancel := loop.flushContext() + defer cancel() + if err := lane.syncBeforeExit(flushCtx, ""); err != nil { + log.Printf("%s: %v", w.label, err) + } + } + return nil + } + } + loop.RunWithWake(ctx, wake) + return nil +} + +// archiveSyncLane tracks when the archive is ready to push and, when this +// process owns the archive, writes it. Watcher batches sync on the watcher's +// dispatch goroutine; the lane goroutine runs authoritative syncs on the floor +// interval, on lifecycle wake, and after the watcher loses coverage. A sync +// that writes the archive marks the push loop dirty. +type archiveSyncLane struct { + label string + // engine is nil when the daemon owns the archive. + engine *syncpkg.Engine + loop *pushLoop + // retry spaces syncs while no complete local sync has run, so a failed + // startup sync does not pause remote pushes for a whole floor interval. + // It doubles after each further failure, up to maxRetry: a retry can + // rebuild the whole archive. + retry, maxRetry time.Duration + startupFailures int + openDispatch func() + requests chan struct{} + // startupSync runs the startup sync, including any archive rebuild it + // needs. The lane retries it until one completes. full is the watch's + // --full request. Local owners only. + startupSync func(context.Context, *syncpkg.Engine, bool) (bool, error) + full bool + // database and startDatabaseID detect a rebuild since the watch started: + // a rebuild gives the archive a new database ID. Local owners only. + database *db.DB + startDatabaseID string + // ready is set once the archive has caught up: after a complete local + // sync, or after the first successful push when the daemon owns the + // archive and syncs before it pushes. Watcher dispatch opens then. A local + // owner also holds pushes until then: a full push deletes remote rows + // missing locally, so it must never read an archive that a failed startup + // sync left incomplete. + ready atomic.Bool + // fullPending asks the next push to resend every session. The first + // push that reaches the destination clears it, even if that push fails: + // each mirror records an unfinished full push and resumes it on the next + // push, and asking for a full push again would start it over. A push that + // fails before it reaches the destination recorded nothing to resume, so + // the request stays. + fullPending atomic.Bool +} + +// markReady opens watcher dispatch the first time the archive catches up and +// reports whether this call did so. +func (l *archiveSyncLane) markReady() bool { + if !l.ready.CompareAndSwap(false, true) { + return false + } + l.openDispatch() + return true +} + +// canPush reports whether a push may run now. A daemon owner always may: its +// push syncs the archive first. +func (l *archiveSyncLane) canPush() bool { + return l.engine == nil || l.ready.Load() +} + +func (l *archiveSyncLane) push( + ctx context.Context, + reason pushReason, + batch *syncpkg.WatchBatch, + push func(context.Context, pushReason, bool, *syncpkg.WatchBatch) error, +) error { + if !l.canPush() { + // Skipping instead of failing keeps the loop from retrying on its + // debounce. The sync that sets ready queues a push. + log.Printf("%s: push (%s) skipped: waiting for a local sync that "+ + "finishes without errors", l.label, reason) + return nil + } + if l.engine != nil && reason == reasonShutdown { + if err := l.syncBeforeExit(ctx, " before the final push"); err != nil { + return err + } + } + if l.engine != nil { + // Wait for an in-progress sync and flush deferred signal recomputes, + // so the push copies settled sessions. Local sync keeps writing + // during the push; those writes reach the next push because each + // push takes its cutoff under the archive write lock. + if err := l.engine.SettleForPush(ctx); err != nil { + return fmt.Errorf("starting push: %w", err) + } + } + full := l.fullPending.Swap(false) + if err := push(ctx, reason, full, batch); err != nil { + if _, notStarted := errors.AsType[storage.PushNotStartedError](err); full && notStarted { + l.fullPending.Store(true) + } + return err + } + if l.engine == nil { + l.markReady() + } + return nil +} + +// requestSync asks the lane for an authoritative sync. Requests coalesce. +func (l *archiveSyncLane) requestSync() { + select { + case l.requests <- struct{}{}: + default: + } +} + +// notifyCoverageDegraded catches the archive up with an authoritative sync +// and logs the loss; the unwatched-root poller takes over the roots. +func (l *archiveSyncLane) notifyCoverageDegraded(roots []string) error { + l.requestSync() + return l.loop.NotifyCoverageDegraded(roots) +} + +func (l *archiveSyncLane) watchCallback(cfg config.Config) syncpkg.WatchCallback { + return func(ctx context.Context, batch syncpkg.WatchBatch) error { + if watchBatchIsAffirmativelyNonData(ctx, cfg, batch) { + return nil + } + err := syncWatchBatch(ctx, l.engine, batch, func() watchRecoveryScope { + return probeWatchRecoveryScope(cfg) + }) + // A failed batch can still have written some sessions. + l.loop.NotifyDirty() + return err + } +} + +func (l *archiveSyncLane) run( + ctx context.Context, tick <-chan time.Time, wake <-chan struct{}, +) { + for { + if !l.wait(ctx, tick, &wake) { + return + } + if l.ready.Load() { + l.syncAll(ctx) + } else if l.syncStartup(ctx) { + // Queue the push that startup skipped. + l.loop.NotifyDirty() + } + } +} + +// wait blocks until the next sync is due and reports false when ctx ends. +func (l *archiveSyncLane) wait( + ctx context.Context, tick <-chan time.Time, wake *<-chan struct{}, +) bool { + var retry <-chan time.Time + if !l.ready.Load() { + timer := time.NewTimer(l.retry) + defer timer.Stop() + retry = timer.C + } + for { + select { + case <-ctx.Done(): + return false + case <-retry: + case <-tick: + case <-l.requests: + case _, ok := <-*wake: + if !ok { + *wake = nil + continue + } + } + return true + } +} + +// syncStartup runs the startup sync and marks the archive ready when it +// completes, reporting whether this call did so. Retries run the same sync, +// so an archive rebuild that an earlier attempt did not finish runs again. +func (l *archiveSyncLane) syncStartup(ctx context.Context) bool { + rebuilt, err := l.archiveRebuilt(ctx) + var didResync bool + if err == nil { + // Once a rebuild has replaced the archive, --full on a retry would + // rebuild it again. + didResync, err = l.startupSync(ctx, l.engine, l.full && !rebuilt) + } + if err == nil { + // A rebuild can replace the archive and still report failures, so + // check again after the sync. + rebuilt, err = l.archiveRebuilt(ctx) + } + if ctx.Err() != nil { + return false + } + if err != nil { + if l.startupFailures > 0 { + l.retry = min(2*l.retry, max(l.maxRetry, l.retry)) + } + l.startupFailures++ + log.Printf("%s: startup sync failed: %v. Remote pushes stay paused. "+ + "Next retry in %s.", l.label, err, l.retry) + return false + } + // Only a full push removes remote rows that a rebuild dropped locally. + if didResync || rebuilt { + l.fullPending.Store(true) + } + return l.markReady() +} + +// archiveRebuilt reports whether a rebuild replaced the archive since the +// watch started. +func (l *archiveSyncLane) archiveRebuilt(ctx context.Context) (bool, error) { + id, err := l.database.GetDatabaseID(ctx) + if err != nil { + return false, fmt.Errorf("reading archive database id: %w", err) + } + return id != l.startDatabaseID, nil +} + +// syncAll runs a periodic sync after the archive is ready. +func (l *archiveSyncLane) syncAll(ctx context.Context) { + stats := l.engine.SyncAll(ctx, nil) + if ctx.Err() != nil { + return + } + switch { + case !stats.AuthoritativeDiscoveryComplete(): + log.Printf("%s: local sync could not list every session. "+ + "The next local sync will try again.", l.label) + case !stats.ProcessingComplete(): + log.Printf("%s: local sync failed to process some sessions. "+ + "The next local sync will try again.", l.label) + } + // A sync that wrote nothing leaves the push loop alone. + if stats.ChangedArchive() { + l.loop.NotifyDirty() + } +} + +// syncBeforeExit runs the last local sync of a stopping watch. The watcher +// has stopped, so changes it had not handed over reach the archive only +// through this sync. what ends the progress line. +func (l *archiveSyncLane) syncBeforeExit(ctx context.Context, what string) error { + log.Printf("%s: stopping. Syncing remaining changes%s.", l.label, what) + l.engine.SyncAll(ctx, nil) + if ctx.Err() != nil { + return errors.New("the final sync did not finish in time. The next " + + "run will pick up changes made since the last sync") + } + return nil +} diff --git a/cmd/agentsview/archive_write_backend.go b/cmd/agentsview/archive_write_backend.go index 14f2d51251..9022390f51 100644 --- a/cmd/agentsview/archive_write_backend.go +++ b/cmd/agentsview/archive_write_backend.go @@ -9,9 +9,11 @@ import ( "os" "sort" stdsync "sync" + "sync/atomic" "time" "go.kenn.io/agentsview/internal/apiclient" + "go.kenn.io/agentsview/internal/archivepush" "go.kenn.io/agentsview/internal/config" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/parser" @@ -79,10 +81,7 @@ type archivePushWatchHooks struct { replicaPush func( context.Context, pushReason, ReplicaPushConfig, ) (storage.PushResult, error) - replicaStartupSync func( - context.Context, *syncpkg.Engine, bool, - ) (bool, error) - duckDBStartupSync func( + startupSync func( context.Context, *syncpkg.Engine, bool, ) (bool, error) newReplicaPusher func(*syncpkg.Engine) *replicaPusher @@ -148,14 +147,14 @@ func newArchivePushLoop( } func archivePushWatchWatcherOptions( - loop *pushLoop, poller unwatchedRootPoller, + notifyCoverageDegraded func([]string) error, poller unwatchedRootPoller, ) syncpkg.WatcherOptions { return syncpkg.WatcherOptions{ OnCoverageDegraded: func(roots []string) error { // Degraded coverage needs both owners: the poller reconciles // the affected roots authoritatively (including tombstoning - // missed deletions) and the loop re-pushes the refreshed - // archive on its floor. + // missed deletions) and the notifier gets the refreshed archive + // pushed. scopes := make([]pollingScope, 0, len(roots)) for _, r := range roots { scopes = append(scopes, pollingScope{Root: r}) @@ -165,7 +164,7 @@ func archivePushWatchWatcherOptions( }); err != nil { return err } - return loop.NotifyCoverageDegraded(roots) + return notifyCoverageDegraded(roots) }, OnPollingRequired: func(obligation syncpkg.PollingObligation) error { return poller.AddObligation(syncObligationToPoller(obligation)) @@ -174,21 +173,18 @@ func archivePushWatchWatcherOptions( } } -func archivePushWatchBatchCallback( - appCfg config.Config, - loop *pushLoop, -) syncpkg.WatchCallback { - return func(callbackCtx context.Context, batch syncpkg.WatchBatch) error { - return notifyPushForWatchBatchWithConfig( - callbackCtx, loop, appCfg, batch, - ) - } -} - func completeDuckDBWatchPush( - res storage.MirrorPushResult, reason pushReason, + res storage.MirrorPushResult, reason pushReason, full bool, ) error { logDuckDBWatchPushResult(res, reason) + if full && res.Diagnostics.Deferred && !res.ArchiveRebuilt { + // A deferred push wrote nothing, so the requested full push is + // still owed. After a daemon rebuilt the archive the mirror pushes + // in full anyway, and asking again would rebuild the archive again. + return storage.PushNotStartedError{ + Err: fmt.Errorf("full push deferred: %s", res.Diagnostics.DeferredReason), + } + } if res.Errors > 0 { return fmt.Errorf("%d session(s) failed to push", res.Errors) } @@ -200,33 +196,12 @@ func completeReplicaWatchPush( ) error { logReplicaWatchPushResult(label, displayName, res, reason) if res.Errors > 0 { + log.Printf("%s: %d session(s) failed to push; will retry", label, res.Errors) return fmt.Errorf("%d session(s) failed to push", res.Errors) } return nil } -func completePushWatchStartup( - ctx context.Context, initialErr error, loop *pushLoop, openDispatch func(), -) { - if initialErr == nil { - if ctx.Err() == nil { - openDispatch() - } - return - } - ack := loop.NotifyDirtyWithAck() - go func() { - select { - case <-ctx.Done(): - return - case err := <-ack: - if err == nil && ctx.Err() == nil { - openDispatch() - } - } - }() -} - func notifyPushForWatchBatch( ctx context.Context, loop *pushLoop, batch syncpkg.WatchBatch, ) error { @@ -472,16 +447,33 @@ func startDaemonPushHeartbeatTo(w io.Writer, label string) func() { // daemonPushProgress pairs a running heartbeat with a streamed-progress // renderer: the heartbeat covers the daemon-side phases that emit no // progress events (local sync, schema migration), and the first streamed -// event silences it for good so heartbeat lines never interleave with the -// in-place progress line. The returned finish func stops the heartbeat (if -// no event ever arrived) and clears the in-place line. +// progress event silences it for good so heartbeat lines never interleave +// with the in-place progress line. A push that waits for another push prints +// one line saying so and keeps the heartbeat. The report that the archive +// was rebuilt also keeps it, since schema migration may follow without +// progress events. A push that waits for a local sync prints a line each time +// the daemon reports the wait. The returned finish func stops the heartbeat (if no event ever +// arrived) and clears the in-place line. func daemonPushProgress[P any]( - label string, render func(P), + label string, phase func(P) string, render func(P), ) (onProgress func(P), finish func()) { stop := startDaemonPushHeartbeat(label) - var once stdsync.Once + var once, waitOnce stdsync.Once stopHeartbeat := func() { once.Do(stop) } onProgress = func(p P) { + switch phase(p) { + case storage.PushPhaseWaiting: + waitOnce.Do(func() { + fmt.Println("Waiting for another push on the daemon to finish...") + }) + return + case storage.PushPhaseRebuilt: + return + case storage.PushPhaseWaitingForSync: + fmt.Print("\r\033[K") + fmt.Println("Waiting for a local sync on the daemon to finish...") + return + } stopHeartbeat() render(p) } @@ -491,6 +483,12 @@ func daemonPushProgress[P any]( } } +// pushProgressPhase and mirrorPushProgressPhase read a progress report's +// phase for daemonPushProgress. +func pushProgressPhase(p storage.PushProgress) string { return p.Phase } + +func mirrorPushProgressPhase(p storage.MirrorPushProgress) string { return p.Phase } + func (b daemonArchiveWriteBackend) ReplicaPush( ctx context.Context, backend storage.Replica, @@ -507,7 +505,7 @@ func (b daemonArchiveWriteBackend) ReplicaPush( return storage.PushResult{}, err } onProgress, finish := daemonPushProgress( - backend.DisplayName(), newReplicaPushProgressPrinter(), + backend.DisplayName(), pushProgressPhase, newReplicaPushProgressPrinter(), ) defer finish() return postDaemonPush[storage.PushResult]( @@ -558,12 +556,8 @@ func (b daemonArchiveWriteBackend) DuckDBPushWatch( if err := requireDaemonSourceSync(ctx, b.appCfg); err != nil { return err } - if interval <= 0 { - interval = defaultWatchInterval - } - if debounce <= 0 { - debounce = defaultWatchDebounce - } + debounce, interval = watchTimings(debounce, interval) + pushWriter := b.watchPushWriter() push := func( pctx context.Context, reason pushReason, full bool, _ *syncpkg.WatchBatch, @@ -580,59 +574,29 @@ func (b daemonArchiveWriteBackend) DuckDBPushWatch( if b.watchHooks != nil && b.watchHooks.duckDBPush != nil { res, err = b.watchHooks.duckDBPush(pctx, reason, full) } else { - backend := archiveWriteBackend(b) - cleanup := func() {} - if reason != reasonStartup { - backend, cleanup, err = resolveArchiveWriteBackend( - pctx, b.appCfg, transportIntentLongLived, - ) - if err != nil { - return err - } + writer, cleanup, werr := pushWriter(pctx) + if werr != nil { + return storage.PushNotStartedError{Err: werr} } defer cleanup() - res, err = backend.DuckDBPush( + res, err = writer.DuckDBPush( pctx, duckCfg, pushCfg, projects, excludeProjects, ) } if err != nil { return err } - return completeDuckDBWatchPush(res, reason) - } - loop, stopLoop := newArchivePushLoop( - b.watchHooks, - "duckdb watch", debounce, interval, - func(c context.Context, r pushReason, batch *syncpkg.WatchBatch) error { - return push(c, r, false, batch) - }, - ) - defer stopLoop() - - stopWatcher, openDispatch, unwatchedDirs := startArchivePushWatcher( - b.watchHooks, b.appCfg, nil, - func(callbackCtx context.Context, batch syncpkg.WatchBatch) error { - return notifyPushForWatchBatchWithConfig( - callbackCtx, loop, b.appCfg, batch, - ) - }, - syncpkg.WatcherOptions{OnCoverageDegraded: loop.NotifyCoverageDegraded}, - ) - defer stopWatcher() - if len(unwatchedDirs) > 0 { - log.Printf( - "duckdb watch: %d root(s) not watched; relying on the %s floor for coverage", - len(unwatchedDirs), interval, - ) - } - initialErr := push(ctx, reasonStartup, cfg.Full, nil) - if initialErr != nil { - log.Printf("duckdb watch: initial daemon push failed: %v", initialErr) - } - completePushWatchStartup(ctx, initialErr, loop, openDispatch) - - loop.Run(ctx) - return nil + return completeDuckDBWatchPush(res, reason, full) + } + return archivePushWatch{ + hooks: b.watchHooks, + appCfg: b.appCfg, + label: "duckdb watch", + debounce: debounce, + interval: interval, + full: cfg.Full, + push: push, + }.run(ctx) } func (b daemonArchiveWriteBackend) duckDBPush( @@ -653,7 +617,7 @@ func (b daemonArchiveWriteBackend) duckDBPush( // non-path fields (machine name, filters) still apply. duckCfg.Path = "" onProgress, finish := daemonPushProgress( - mirrorBackend.DisplayName(), func(p storage.MirrorPushProgress) { + mirrorBackend.DisplayName(), mirrorPushProgressPhase, func(p storage.MirrorPushProgress) { fmt.Printf( "\rPushing... %d/%d sessions, %d messages\x1b[K", p.SessionsDone, p.SessionsTotal, p.MessagesDone, @@ -699,92 +663,74 @@ func (b daemonArchiveWriteBackend) ReplicaPushWatch( if err := requireDaemonSourceSync(ctx, b.appCfg); err != nil { return err } - if interval <= 0 { - interval = defaultWatchInterval - } - if debounce <= 0 { - debounce = defaultWatchDebounce - } - // Daemon-delegated pushes build a fresh pusher per request, so the - // vector reconcile bit and the last-reconciled generation id live - // here, in the long-lived watch process, mirroring replicaPusher's - // local-mode state. + debounce, interval = watchTimings(debounce, interval) + pushWriter := b.watchPushWriter() label := backend.Name() + " watch" - vectorReconcileNeeded := true - lastReconciledVectorGeneration := int64(0) + vectors := newVectorReconcile() push := func( pctx context.Context, reason pushReason, full bool, batch *syncpkg.WatchBatch, ) error { - pushCfg := cfg - pushCfg.Full = full - pushCfg.WatchBatch = batch - pushCfg.WatchRecovery = watchRecoveryForBatch(b.appCfg, batch) - scoped := scopedVectorPush(reason, full, vectorReconcileNeeded) - pushCfg.ScopeVectorsToChangedSessions = scoped - pushCfg.LastReconciledVectorGeneration = lastReconciledVectorGeneration - var res storage.PushResult - var err error - if b.watchHooks != nil && b.watchHooks.replicaPush != nil { - res, err = b.watchHooks.replicaPush(pctx, reason, pushCfg) - } else { - writer := archiveWriteBackend(b) - cleanup := func() {} - if reason != reasonStartup { - writer, cleanup, err = resolveArchiveWriteBackend( - pctx, b.appCfg, transportIntentLongLived, - ) + return vectors.push( + label, backend.DisplayName(), reason, full, + func(opts storage.PushOptions) (storage.PushResult, error) { + pushCfg := cfg + pushCfg.Full = full + pushCfg.WatchBatch = batch + pushCfg.WatchRecovery = watchRecoveryForBatch(b.appCfg, batch) + pushCfg.ScopeVectorsToChangedSessions = opts.ScopeVectorsToChangedSessions + pushCfg.LastReconciledVectorGeneration = opts.LastReconciledVectorGeneration + if b.watchHooks != nil && b.watchHooks.replicaPush != nil { + return b.watchHooks.replicaPush(pctx, reason, pushCfg) + } + writer, cleanup, err := pushWriter(pctx) if err != nil { - return err + return storage.PushResult{}, storage.PushNotStartedError{Err: err} } - } - defer cleanup() - res, err = writer.ReplicaPush( - pctx, backend, target, pushCfg, projects, exclude, - ) - } - if err != nil { - vectorReconcileNeeded = true - return err - } - vectorReconcileNeeded, lastReconciledVectorGeneration = nextVectorReconcile( - vectorReconcileNeeded, - lastReconciledVectorGeneration, scoped, res, + defer cleanup() + return writer.ReplicaPush( + pctx, backend, target, pushCfg, projects, exclude, + ) + }, ) - return completeReplicaWatchPush(label, backend.DisplayName(), res, reason) } - loop, stopLoop := newArchivePushLoop( - b.watchHooks, label, debounce, interval, - func(c context.Context, r pushReason, batch *syncpkg.WatchBatch) error { - return push(c, r, false, batch) - }, - ) - defer stopLoop() + return archivePushWatch{ + hooks: b.watchHooks, + appCfg: b.appCfg, + label: label, + debounce: debounce, + interval: interval, + full: cfg.Full, + push: push, + wake: cfg.LifecycleWake, + }.run(ctx) +} - stopWatcher, openDispatch, unwatchedDirs := startArchivePushWatcher( - b.watchHooks, b.appCfg, nil, - func(callbackCtx context.Context, batch syncpkg.WatchBatch) error { - return notifyPushForWatchBatchWithConfig( - callbackCtx, loop, b.appCfg, batch, - ) - }, - syncpkg.WatcherOptions{OnCoverageDegraded: loop.NotifyCoverageDegraded}, - ) - defer stopWatcher() - if len(unwatchedDirs) > 0 { - log.Printf( - "pg watch: %d root(s) not watched; relying on the %s floor for coverage", - len(unwatchedDirs), interval, - ) - } - initialErr := push(ctx, reasonStartup, cfg.Full, nil) - if initialErr != nil { - log.Printf("pg watch: initial daemon push failed: %v", initialErr) +// watchPushWriter returns the writer source for one daemon watch. The first +// push reuses the connection the watch started with. Every later push, +// including a retry of a failed startup push, resolves the archive owner +// again, because the daemon may have restarted on another address. +func (b daemonArchiveWriteBackend) watchPushWriter() func( + context.Context, +) (archiveWriteBackend, func(), error) { + var used atomic.Bool + return func(ctx context.Context) (archiveWriteBackend, func(), error) { + if used.CompareAndSwap(false, true) { + return b, func() {}, nil + } + return resolveArchiveWriteBackend(ctx, b.appCfg, transportIntentLongLived) } - completePushWatchStartup(ctx, initialErr, loop, openDispatch) +} - loop.RunWithWake(ctx, cfg.LifecycleWake) - return nil +// watchTimings applies the default debounce and floor interval. +func watchTimings(debounce, interval time.Duration) (time.Duration, time.Duration) { + if debounce <= 0 { + debounce = defaultWatchDebounce + } + if interval <= 0 { + interval = defaultWatchInterval + } + return debounce, interval } type localArchiveWriteBackend struct { @@ -805,18 +751,16 @@ func (b *localArchiveWriteBackend) ensureCurrentPricing( func (b *localArchiveWriteBackend) newReplicaPusher( backend storage.Replica, - localSync func(context.Context) error, connect func(context.Context) (storage.Pusher, error), ) *replicaPusher { return &replicaPusher{ - label: backend.Name() + " watch", - displayName: backend.DisplayName(), - localSync: localSync, - ensurePricing: b.ensureCurrentPricing, - connect: connect, - // True until the startup push completes a clean - // generation-wide vector reconciliation. - vectorReconcileNeeded: true, + label: backend.Name() + " watch", + displayName: backend.DisplayName(), + session: archivepush.ReplicaSession{ + Connect: connect, + EnsurePricing: b.ensureCurrentPricing, + }, + vectors: newVectorReconcile(), } } @@ -831,7 +775,6 @@ func (b *localArchiveWriteBackend) ReplicaPush( if err := b.database.RequireSourceSync(ctx); err != nil { return storage.PushResult{}, err } - display := backend.DisplayName() var embedder *replicaEmbedder if cfg.Embed { var err error @@ -859,16 +802,6 @@ func (b *localArchiveWriteBackend) ReplicaPush( if err := ctx.Err(); err != nil { return storage.PushResult{}, err } - if err := b.ensureCurrentPricing(ctx); err != nil { - if ctxErr := ctx.Err(); ctxErr != nil { - return storage.PushResult{}, ctxErr - } - log.Printf("warning: pricing refresh failed: %v", err) - } - if err := ctx.Err(); err != nil { - return storage.PushResult{}, err - } - forceFull := cfg.Full || didResync var vectorSource storage.VectorPushSource if embedder != nil { @@ -877,55 +810,54 @@ func (b *localArchiveWriteBackend) ReplicaPush( vectorSource = replicaVectorPushSource(b.appCfg, target, cfg) defer closeVectorPushSource(vectorSource) } - - fmt.Printf("Connecting to %s...\n", display) - connectStart := time.Now() - applyClassifierConfig(b.appCfg) - ps, err := backend.NewPusher( - ctx, target.Target, b.database, - replicaPusherOptions(target, projects, excludeProjects, vectorSource), - ) - if err != nil { - return storage.PushResult{}, err - } - defer ps.Close() - fmt.Printf( - "Connected to %s in %s\n", display, - time.Since(connectStart).Round(time.Millisecond), - ) - - fmt.Printf("Preparing %s schema...\n", display) - schemaStart := time.Now() - if err := ps.EnsureSchema(ctx); err != nil { - return storage.PushResult{}, fmt.Errorf("schema: %w", err) - } - fmt.Printf( - "%s schema ready in %s\n", display, - time.Since(schemaStart).Round(time.Millisecond), - ) - fmt.Printf("Starting %s push...\n", display) - result, err := ps.PushWithOptions(ctx, storage.PushOptions{ - Full: forceFull, - ScopeVectorsToChangedSessions: cfg. - ScopeVectorsToChangedSessions, - LastReconciledVectorGeneration: cfg. - LastReconciledVectorGeneration, + session := archivepush.ReplicaSession{ + Connect: func(c context.Context) (storage.Pusher, error) { + applyClassifierConfig(b.appCfg) + return backend.NewPusher( + c, target.Target, b.database, + replicaPusherOptions(target, projects, excludeProjects, vectorSource), + ) + }, + EnsurePricing: b.ensureCurrentPricing, + OnStage: printReplicaPushStage(backend.DisplayName()), + } + defer session.Close() + result, err := session.Push(ctx, storage.PushOptions{ + Full: cfg.Full || didResync, + ScopeVectorsToChangedSessions: cfg.ScopeVectorsToChangedSessions, + LastReconciledVectorGeneration: cfg.LastReconciledVectorGeneration, }, newReplicaPushProgressPrinter()) fmt.Print("\r\033[K") return result, err } -func (b *localArchiveWriteBackend) DuckDBPush( - ctx context.Context, - duckCfg config.DuckDBConfig, - cfg DuckDBPushConfig, - projects []string, - excludeProjects []string, -) (storage.MirrorPushResult, error) { - return b.duckDBPush(ctx, duckCfg, cfg, projects, excludeProjects) +// printReplicaPushStage reports each replica push stage on stdout, with the +// time the connect and schema stages took. +func printReplicaPushStage(display string) func(archivepush.Stage) func() { + return func(stage archivepush.Stage) func() { + start := time.Now() + elapsed := func() time.Duration { + return time.Since(start).Round(time.Millisecond) + } + switch stage { + case archivepush.StageConnect: + fmt.Printf("Connecting to %s...\n", display) + return func() { + fmt.Printf("Connected to %s in %s\n", display, elapsed()) + } + case archivepush.StageSchema: + fmt.Printf("Preparing %s schema...\n", display) + return func() { + fmt.Printf("%s schema ready in %s\n", display, elapsed()) + } + case archivepush.StagePush: + fmt.Printf("Starting %s push...\n", display) + } + return nil + } } -func (b *localArchiveWriteBackend) duckDBPush( +func (b *localArchiveWriteBackend) DuckDBPush( ctx context.Context, duckCfg config.DuckDBConfig, cfg DuckDBPushConfig, @@ -944,11 +876,10 @@ func (b *localArchiveWriteBackend) duckDBPush( if err != nil { return storage.MirrorPushResult{}, err } - forceFull := cfg.Full || didResync fmt.Println("Starting DuckDB push...") return b.duckDBMirrorPush( - ctx, duckCfg, cfg, projects, excludeProjects, forceFull, + ctx, duckCfg, cfg, projects, excludeProjects, cfg.Full || didResync, ) } @@ -963,20 +894,22 @@ func (b *localArchiveWriteBackend) duckDBMirrorPush( if err := mirrorBackend.ValidatePushTarget(duckCfg); err != nil { return storage.MirrorPushResult{}, err } - opts := storage.MirrorPushOptions{ - Projects: projects, - ExcludeProjects: excludeProjects, - Automatic: cfg.Automatic, - } - result, err := mirrorBackend.Push( - ctx, duckCfg, b.database, opts, forceFull, - func(p storage.MirrorPushProgress) { - fmt.Printf( - "\rPushing... %d/%d sessions, %d messages\x1b[K", - p.SessionsDone, p.SessionsTotal, p.MessagesDone, - ) + result, err := archivepush.MirrorPush{ + Mirror: mirrorBackend, + Config: duckCfg, + Archive: b.database, + Options: storage.MirrorPushOptions{ + Projects: projects, + ExcludeProjects: excludeProjects, + Automatic: cfg.Automatic, }, - ) + EnsurePricing: b.ensureCurrentPricing, + }.Push(ctx, forceFull, func(p storage.MirrorPushProgress) { + fmt.Printf( + "\rPushing... %d/%d sessions, %d messages\x1b[K", + p.SessionsDone, p.SessionsTotal, p.MessagesDone, + ) + }) fmt.Print("\r\033[K") if err != nil { return storage.MirrorPushResult{}, err @@ -985,7 +918,6 @@ func (b *localArchiveWriteBackend) duckDBMirrorPush( } func (b *localArchiveWriteBackend) newDuckDBPusher( - engine *syncpkg.Engine, duckCfg config.DuckDBConfig, cfg DuckDBPushConfig, projects, exclude []string, @@ -993,21 +925,6 @@ func (b *localArchiveWriteBackend) newDuckDBPusher( pushCfg := cfg pushCfg.Automatic = true return &duckDBPusher{ - localSync: func(c context.Context) error { - stats := engine.SyncAll(c, nil) - if err := c.Err(); err != nil { - return err - } - if !stats.AuthoritativeDiscoveryComplete() { - return errors.New("local sync discovery incomplete") - } - if !stats.ProcessingComplete() { - return errors.New("local sync processing incomplete") - } - engine.FlushSignals() - return nil - }, - ensurePricing: b.ensureCurrentPricing, mirrorPush: func(c context.Context, forceFull bool) ( storage.MirrorPushResult, error, ) { @@ -1027,105 +944,34 @@ func (b *localArchiveWriteBackend) DuckDBPushWatch( debounce time.Duration, interval time.Duration, ) error { - if err := b.database.RequireSourceSync(ctx); err != nil { + engine, err := b.startWatchEngine(ctx, nil) + if err != nil { if ctx.Err() != nil { return nil } return err } - if interval <= 0 { - interval = defaultWatchInterval - } - if debounce <= 0 { - debounce = defaultWatchDebounce - } - for _, def := range parser.Registry { - if !b.appCfg.IsUserConfigured(def.Type) { - continue - } - warnMissingDirs(b.appCfg.ResolveDirs(def.Type), string(def.Type)) - } - cleanResyncTemp(b.appCfg.DBPath) - - engine := syncpkg.NewEngine(ctx, b.database, syncpkg.EngineConfig{ - AgentDirs: b.appCfg.AgentDirs, - SourceMachines: b.appCfg.SourceMachines, - ProviderMetadata: b.appCfg.ProviderMetadata, - DisabledAgents: b.appCfg.DisabledAgents, - IncludeCwdPrefixes: b.appCfg.SyncIncludeCwdPrefixes, - ScanProtectedPaths: b.appCfg.ScanProtectedPaths, - Machine: b.appCfg.InstallationID, - BlockedResultCategories: b.appCfg.ResultContentBlockedCategories, - ArchiveContent: b.appCfg.ArchiveContent, - FrictionEnabled: b.appCfg.Friction.Enabled, - }) defer engine.Close() var pusher *duckDBPusher if b.watchHooks != nil && b.watchHooks.newDuckDBPusher != nil { pusher = b.watchHooks.newDuckDBPusher(engine) } else { - pusher = b.newDuckDBPusher(engine, duckCfg, cfg, projects, exclude) - } - pusher.scopedSync = func( - c context.Context, - batch syncpkg.WatchBatch, - recovery *syncpkg.WatchRecoveryScope, - work func() error, - ) error { - _, err := engine.SyncWatchBatchThenRun(c, batch, recovery, work) - return err - } - - loop, stopLoop := newArchivePushLoop( - b.watchHooks, - "duckdb watch", debounce, interval, - func(c context.Context, r pushReason, batch *syncpkg.WatchBatch) error { - return pusher.pushBatch( - c, r, false, batch, watchRecoveryForBatch(b.appCfg, batch), - ) - }, - ) - defer stopLoop() - - poller := newArchivePushUnwatchedPoller(ctx, b.watchHooks, engine) - defer poller.Stop() - - stopWatcher, openDispatch, unwatchedDirs := startArchivePushWatcher( - b.watchHooks, b.appCfg, engine, - archivePushWatchBatchCallback(b.appCfg, loop), - archivePushWatchWatcherOptions(loop, poller), - ) - defer stopWatcher() - if len(unwatchedDirs) > 0 { - log.Printf( - "duckdb watch: %d root(s) not watched; polling every %s", - len(unwatchedDirs), unwatchedPollInterval, - ) - } - - startupSync := runPGWatchStartupSync - if b.watchHooks != nil && b.watchHooks.duckDBStartupSync != nil { - startupSync = b.watchHooks.duckDBStartupSync - } - didResync, startupErr := startupSync(ctx, engine, cfg.Full) - if startupErr != nil && errors.Is(startupErr, context.Canceled) { - return nil - } - initialErr := startupErr - if initialErr == nil { - initialErr = pusher.push(ctx, reasonStartup, didResync) - } - if initialErr != nil { - if errors.Is(initialErr, context.Canceled) && ctx.Err() != nil { - return nil - } - log.Printf("duckdb watch: initial push failed: %v", initialErr) - } - completePushWatchStartup(ctx, initialErr, loop, openDispatch) - - loop.Run(ctx) - return nil + pusher = b.newDuckDBPusher(duckCfg, cfg, projects, exclude) + } + + debounce, interval = watchTimings(debounce, interval) + return archivePushWatch{ + hooks: b.watchHooks, + appCfg: b.appCfg, + engine: engine, + database: b.database, + label: "duckdb watch", + debounce: debounce, + interval: interval, + full: cfg.Full, + push: ignoreWatchBatch(pusher.push), + }.run(ctx) } func logDuckDBWatchPushResult(res storage.MirrorPushResult, reason pushReason) { @@ -1168,26 +1014,7 @@ func (b *localArchiveWriteBackend) ReplicaPushWatch( debounce time.Duration, interval time.Duration, ) error { - if err := b.database.RequireSourceSync(ctx); err != nil { - if ctx.Err() != nil { - return nil - } - return err - } - if interval <= 0 { - interval = defaultWatchInterval - } - if debounce <= 0 { - debounce = defaultWatchDebounce - } - for _, def := range parser.Registry { - if !b.appCfg.IsUserConfigured(def.Type) { - continue - } - warnMissingDirs(b.appCfg.ResolveDirs(def.Type), string(def.Type)) - } - cleanResyncTemp(b.appCfg.DBPath) - + debounce, interval = watchTimings(debounce, interval) var embedder *replicaEmbedder var emitter syncpkg.Emitter if cfg.Embed { @@ -1198,20 +1025,13 @@ func (b *localArchiveWriteBackend) ReplicaPushWatch( embedder.resolveRetry = interval emitter = embedder } - - engine := syncpkg.NewEngine(ctx, b.database, syncpkg.EngineConfig{ - Emitter: emitter, - AgentDirs: b.appCfg.AgentDirs, - SourceMachines: b.appCfg.SourceMachines, - ProviderMetadata: b.appCfg.ProviderMetadata, - DisabledAgents: b.appCfg.DisabledAgents, - IncludeCwdPrefixes: b.appCfg.SyncIncludeCwdPrefixes, - ScanProtectedPaths: b.appCfg.ScanProtectedPaths, - Machine: b.appCfg.InstallationID, - BlockedResultCategories: b.appCfg.ResultContentBlockedCategories, - ArchiveContent: b.appCfg.ArchiveContent, - FrictionEnabled: b.appCfg.Friction.Enabled, - }) + engine, err := b.startWatchEngine(ctx, emitter) + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } defer engine.Close() name := backend.Name() @@ -1236,26 +1056,6 @@ func (b *localArchiveWriteBackend) ReplicaPushWatch( defer closeVectorPushSource(vectorSource) pusher = b.newReplicaPusher( backend, - func(c context.Context) error { - stats := engine.SyncAll(c, nil) - if err := c.Err(); err != nil { - return err - } - if !stats.AuthoritativeDiscoveryComplete() { - return errors.New("local sync discovery incomplete") - } - if !stats.ProcessingComplete() { - return errors.New("local sync processing incomplete") - } - // The push scans SQLite rows right after this returns; - // flush deferred signal recomputes and stale friction so - // pushed sessions carry current derived fields. - engine.FlushSignals() - if storage.StoresFriction(backend) { - drainStaleFriction(c, engine) - } - return nil - }, func(c context.Context) (storage.Pusher, error) { applyClassifierConfig(b.appCfg) return backend.NewPusher( @@ -1265,24 +1065,11 @@ func (b *localArchiveWriteBackend) ReplicaPushWatch( }, ) } - pusher.scopedSync = func( - c context.Context, - batch syncpkg.WatchBatch, - recovery *syncpkg.WatchRecoveryScope, - work func() error, - ) error { - _, err := engine.SyncWatchBatchThenRun(c, batch, recovery, func() error { - if storage.StoresFriction(backend) { - if err := engine.DrainStaleFrictionLocked(c); err != nil { - return err - } - } - return work() - }) - return err + if storage.StoresFriction(backend) { + pusher.session.FrictionReview = engine } - pusher.fullVectorsPending = cfg.Full - defer pusher.reset() + pusher.vectors.fullPending = cfg.Full + defer pusher.session.Close() fmt.Printf( "agentsview %s watch: pushing to %s as %q "+ @@ -1290,61 +1077,70 @@ func (b *localArchiveWriteBackend) ReplicaPushWatch( name, backend.DisplayName(), target.Target.MachineName, debounce, interval, ) - loop, stopLoop := newArchivePushLoop( - b.watchHooks, name+" watch", debounce, interval, - func(c context.Context, r pushReason, batch *syncpkg.WatchBatch) error { - err := pusher.pushBatch( - c, r, false, batch, watchRecoveryForBatch(b.appCfg, batch), - ) - if embedder != nil && r != reasonShutdown { + push := pusher.push + if embedder != nil { + // The embedder's scheduler state belongs to the push loop, which + // runs every push, so it starts the scheduler there. + push = func(c context.Context, r pushReason, full bool) error { + if r != reasonShutdown { embedder.ensureScheduler(ctx) } - return err - }, - ) - defer stopLoop() - - poller := newArchivePushUnwatchedPoller(ctx, b.watchHooks, engine) - defer poller.Stop() - - stopWatcher, openDispatch, unwatchedDirs := startArchivePushWatcher( - b.watchHooks, b.appCfg, engine, - archivePushWatchBatchCallback(b.appCfg, loop), - archivePushWatchWatcherOptions(loop, poller), - ) - defer stopWatcher() - if len(unwatchedDirs) > 0 { - log.Printf( - "pg watch: %d root(s) not watched; polling every %s", - len(unwatchedDirs), unwatchedPollInterval, - ) + return pusher.push(c, r, full) + } } + return archivePushWatch{ + hooks: b.watchHooks, + appCfg: b.appCfg, + engine: engine, + database: b.database, + label: name + " watch", + debounce: debounce, + interval: interval, + full: cfg.Full, + push: ignoreWatchBatch(push), + wake: cfg.LifecycleWake, + }.run(ctx) +} - startupSync := runPGWatchStartupSync - if b.watchHooks != nil && b.watchHooks.replicaStartupSync != nil { - startupSync = b.watchHooks.replicaStartupSync - } - didResync, startupErr := startupSync(ctx, engine, cfg.Full) - if startupErr != nil && errors.Is(startupErr, context.Canceled) { - return nil - } - initialErr := startupErr - if embedder != nil { - embedder.ensureScheduler(ctx) - } - if initialErr == nil { - initialErr = pusher.push(ctx, reasonStartup, didResync) +// startWatchEngine prepares a watch that owns the archive and returns the sync +// engine that writes it. +func (b *localArchiveWriteBackend) startWatchEngine( + ctx context.Context, emitter syncpkg.Emitter, +) (*syncpkg.Engine, error) { + if err := b.database.RequireSourceSync(ctx); err != nil { + return nil, err } - if initialErr != nil { - if errors.Is(initialErr, context.Canceled) && ctx.Err() != nil { - return nil + for _, def := range parser.Registry { + if !b.appCfg.IsUserConfigured(def.Type) { + continue } - log.Printf("pg watch: initial push failed: %v", initialErr) + warnMissingDirs(b.appCfg.ResolveDirs(def.Type), string(def.Type)) } - completePushWatchStartup(ctx, initialErr, loop, openDispatch) + cleanResyncTemp(b.appCfg.DBPath) + return syncpkg.NewEngine(ctx, b.database, syncpkg.EngineConfig{ + Emitter: emitter, + AgentDirs: b.appCfg.AgentDirs, + SourceMachines: b.appCfg.SourceMachines, + ProviderMetadata: b.appCfg.ProviderMetadata, + DisabledAgents: b.appCfg.DisabledAgents, + IncludeCwdPrefixes: b.appCfg.SyncIncludeCwdPrefixes, + ScanProtectedPaths: b.appCfg.ScanProtectedPaths, + Machine: b.appCfg.InstallationID, + BlockedResultCategories: b.appCfg.ResultContentBlockedCategories, + ArchiveContent: b.appCfg.ArchiveContent, + FrictionEnabled: b.appCfg.Friction.Enabled, + }), nil +} - loop.RunWithWake(ctx, cfg.LifecycleWake) - return nil +// ignoreWatchBatch adapts a local owner's pusher to the watch loop. A local +// owner syncs each watcher batch before it queues a push, so the push needs +// only the archive. +func ignoreWatchBatch( + push func(context.Context, pushReason, bool) error, +) func(context.Context, pushReason, bool, *syncpkg.WatchBatch) error { + return func(ctx context.Context, reason pushReason, full bool, _ *syncpkg.WatchBatch) error { + return push(ctx, reason, full) + } } func runPGWatchStartupSync( diff --git a/cmd/agentsview/archive_write_backend_test.go b/cmd/agentsview/archive_write_backend_test.go index 642d2ed201..c88119df82 100644 --- a/cmd/agentsview/archive_write_backend_test.go +++ b/cmd/agentsview/archive_write_backend_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "strings" stdsync "sync" "testing" "testing/synctest" @@ -14,6 +15,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/archivepush" "go.kenn.io/agentsview/internal/config" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/dbtest" @@ -65,7 +67,7 @@ func TestDaemonPushProgressSilencesHeartbeat(t *testing.T) { t.Cleanup(func() { daemonPushHeartbeatInterval = orig }) onProgress, finish := daemonPushProgress( - "PostgreSQL", newReplicaPushProgressPrinter(), + "PostgreSQL", pushProgressPhase, newReplicaPushProgressPrinter(), ) onProgress(storage.PushProgress{SessionsDone: 1, SessionsTotal: 2}) synctest.Sleep(150 * time.Millisecond) @@ -79,6 +81,49 @@ func TestDaemonPushProgressSilencesHeartbeat(t *testing.T) { "heartbeat must stay silent after the first progress event") } +// A push waiting for another push on the daemon says so once. The heartbeat +// keeps running, because the wait can be long and sends no progress. +func TestDaemonPushProgressReportsWaitingForAnotherPush(t *testing.T) { + out := captureStdout(t, func() { + synctest.Test(t, func(t *testing.T) { + orig := daemonPushHeartbeatInterval + daemonPushHeartbeatInterval = 50 * time.Millisecond + t.Cleanup(func() { daemonPushHeartbeatInterval = orig }) + + onProgress, finish := daemonPushProgress( + "DuckDB", mirrorPushProgressPhase, + func(storage.MirrorPushProgress) { + assert.Fail(t, "a waiting report is not push progress") + }, + ) + onProgress(storage.MirrorPushProgress{Phase: storage.PushPhaseWaiting}) + onProgress(storage.MirrorPushProgress{Phase: storage.PushPhaseWaiting}) + synctest.Sleep(75 * time.Millisecond) + finish() + }) + }) + + assert.Equal(t, 1, strings.Count(out, "Waiting for another push on the daemon to finish")) + assert.Contains(t, out, "still pushing to DuckDB via the daemon") +} + +// A push that waits on a local sync says so each time the daemon reports the +// wait, and the report is not push progress. +func TestDaemonPushProgressReportsWaitingForTheLocalSync(t *testing.T) { + out := captureStdout(t, func() { + onProgress, finish := daemonPushProgress( + "DuckDB", mirrorPushProgressPhase, + func(storage.MirrorPushProgress) { + assert.Fail(t, "a waiting report is not push progress") + }, + ) + onProgress(storage.MirrorPushProgress{Phase: storage.PushPhaseWaitingForSync}) + finish() + }) + + assert.Equal(t, 1, strings.Count(out, "Waiting for a local sync on the daemon to finish")) +} + // syncBuffer is a mutex-guarded bytes.Buffer: the heartbeat goroutine writes // while the test reads. type syncBuffer struct { @@ -143,7 +188,6 @@ func TestLocalPGWatchPusherUsesBackendPricingEnsure(t *testing.T) { target := &fakeTarget{} pusher := backend.newReplicaPusher( pgReplica{}, - func(context.Context) error { return nil }, func(context.Context) (storage.Pusher, error) { return target, nil }, ) @@ -368,6 +412,10 @@ func TestLocalArchiveWriteBackendPGPushWatchCanceledStartupIsClean(t *testing.T) type pushWatchOwnerCase struct { name string run func(context.Context, *archivePushWatchHooks) error + // runFull runs the watch with --full. Set for local owners only. + runFull func(context.Context, *archivePushWatchHooks) error + // database is the archive a local owner writes. Set for local owners only. + database *db.DB } func pushWatchOwnerCases(t *testing.T) []pushWatchOwnerCase { @@ -395,26 +443,39 @@ func pushWatchOwnerCases(t *testing.T) []pushWatchOwnerCase { ) }, }, - { - name: "local DuckDB", - run: func(ctx context.Context, hooks *archivePushWatchHooks) error { - localDuckDB.watchHooks = hooks - return localDuckDB.DuckDBPushWatch( - ctx, config.DuckDBConfig{}, DuckDBPushConfig{}, nil, nil, - time.Hour, time.Hour, - ) - }, - }, - { - name: "local PostgreSQL", - run: func(ctx context.Context, hooks *archivePushWatchHooks) error { - localPostgreSQL.watchHooks = hooks - return localPostgreSQL.ReplicaPushWatch( - ctx, pgReplica{}, storage.ConfiguredReplica{}, ReplicaPushConfig{}, nil, nil, - time.Hour, time.Hour, - ) - }, - }, + localDuckDBCase(localDuckDB), + localPostgreSQLCase(localPostgreSQL), + } +} + +func localDuckDBCase(backend *localArchiveWriteBackend) pushWatchOwnerCase { + run := func(full bool) func(context.Context, *archivePushWatchHooks) error { + return func(ctx context.Context, hooks *archivePushWatchHooks) error { + backend.watchHooks = hooks + return backend.DuckDBPushWatch( + ctx, config.DuckDBConfig{}, DuckDBPushConfig{Full: full}, nil, nil, + localPushWatchTestDebounce, time.Hour, + ) + } + } + return pushWatchOwnerCase{ + name: "local DuckDB", run: run(false), runFull: run(true), database: backend.database, + } +} + +func localPostgreSQLCase(backend *localArchiveWriteBackend) pushWatchOwnerCase { + run := func(full bool) func(context.Context, *archivePushWatchHooks) error { + return func(ctx context.Context, hooks *archivePushWatchHooks) error { + backend.watchHooks = hooks + return backend.ReplicaPushWatch( + ctx, pgReplica{}, storage.ConfiguredReplica{}, + ReplicaPushConfig{Full: full}, nil, nil, + localPushWatchTestDebounce, time.Hour, + ) + } + } + return pushWatchOwnerCase{ + name: "local PostgreSQL", run: run(false), runFull: run(true), database: backend.database, } } @@ -520,25 +581,25 @@ func TestDaemonPGPushWatchRenameProbesDeferredRecoveryAtExecution(t *testing.T) type pushWatchAttempt struct { index int reason pushReason + full bool batch *syncpkg.WatchBatch recovery *syncpkg.WatchRecoveryScope } type pushWatchOwnerHarness struct { - mu stdsync.Mutex - partial map[int]bool - events []string - attempts chan pushWatchAttempt - opened chan struct{} - callback syncpkg.WatchCallback - degraded func([]string) error - loop *pushLoop - fire chan time.Time - floor chan time.Time - openCount int - startupSyncs int - localPushSyncs int - attemptCount int + mu stdsync.Mutex + partial map[int]bool + events []string + attempts chan pushWatchAttempt + opened chan struct{} + callback syncpkg.WatchCallback + degraded func([]string) error + loop *pushLoop + fire chan time.Time + floor chan time.Time + openCount int + startupSyncs int + attemptCount int } func newPushWatchOwnerHarness(partial ...int) *pushWatchOwnerHarness { @@ -606,16 +667,7 @@ func (h *pushWatchOwnerHarness) hooks() *archivePushWatchHooks { } return storage.PushResult{SessionsPushed: attempt}, nil }, - replicaStartupSync: func( - context.Context, *syncpkg.Engine, bool, - ) (bool, error) { - h.mu.Lock() - h.startupSyncs++ - h.events = append(h.events, "startup-sync") - h.mu.Unlock() - return false, nil - }, - duckDBStartupSync: func( + startupSync: func( context.Context, *syncpkg.Engine, bool, ) (bool, error) { h.mu.Lock() @@ -627,29 +679,17 @@ func (h *pushWatchOwnerHarness) hooks() *archivePushWatchHooks { newReplicaPusher: func(*syncpkg.Engine) *replicaPusher { target := &pushWatchPGTarget{harness: h} return &replicaPusher{ - localSync: func(context.Context) error { - h.mu.Lock() - h.localPushSyncs++ - h.events = append(h.events, "local-sync") - h.mu.Unlock() - return nil + session: archivepush.ReplicaSession{ + Connect: func(context.Context) (storage.Pusher, error) { return target, nil }, }, - connect: func(context.Context) (storage.Pusher, error) { return target, nil }, } }, newDuckDBPusher: func(*syncpkg.Engine) *duckDBPusher { return &duckDBPusher{ - localSync: func(context.Context) error { - h.mu.Lock() - h.localPushSyncs++ - h.events = append(h.events, "local-sync") - h.mu.Unlock() - return nil - }, mirrorPush: func( - _ context.Context, _ bool, + _ context.Context, full bool, ) (storage.MirrorPushResult, error) { - attempt, partial := h.nextAttempt("") + attempt, partial := h.nextLocalAttempt(full) if partial { return storage.MirrorPushResult{Errors: 1}, nil } @@ -672,10 +712,12 @@ func (h *pushWatchOwnerHarness) nextAttempt(reason pushReason) (int, bool) { return h.nextAttemptLocked(reason) } -func (h *pushWatchOwnerHarness) nextPGAttempt() (int, bool) { +// nextLocalAttempt records a push from a local owner's pusher, which sees the +// full flag but not the loop's reason. +func (h *pushWatchOwnerHarness) nextLocalAttempt(full bool) (int, bool) { h.mu.Lock() defer h.mu.Unlock() - return h.nextAttemptLocked("") + return h.nextAttemptWithScopeLocked("", full, nil, nil) } func (h *pushWatchOwnerHarness) nextPGWatchAttempt( @@ -683,17 +725,20 @@ func (h *pushWatchOwnerHarness) nextPGWatchAttempt( ) (int, bool) { h.mu.Lock() defer h.mu.Unlock() - return h.nextAttemptWithScopeLocked(reason, cfg.WatchBatch, cfg.WatchRecovery) + return h.nextAttemptWithScopeLocked( + reason, false, cfg.WatchBatch, cfg.WatchRecovery, + ) } func (h *pushWatchOwnerHarness) nextAttemptLocked( reason pushReason, ) (int, bool) { - return h.nextAttemptWithScopeLocked(reason, nil, nil) + return h.nextAttemptWithScopeLocked(reason, false, nil, nil) } func (h *pushWatchOwnerHarness) nextAttemptWithScopeLocked( reason pushReason, + full bool, batch *syncpkg.WatchBatch, recovery *syncpkg.WatchRecoveryScope, ) (int, bool) { @@ -702,7 +747,8 @@ func (h *pushWatchOwnerHarness) nextAttemptWithScopeLocked( h.events = append(h.events, "push") partial := h.partial[index] h.attempts <- pushWatchAttempt{ - index: index, reason: reason, batch: batch, recovery: recovery, + index: index, reason: reason, full: full, batch: batch, + recovery: recovery, } return index, partial } @@ -721,13 +767,10 @@ func pushLoopPendingState(loop *pushLoop) (bool, int) { return len(loop.pendingTasks) > 0, waiters } -func (h *pushWatchOwnerHarness) snapshot() ( - []string, int, int, int, -) { +func (h *pushWatchOwnerHarness) snapshot() ([]string, int, int) { h.mu.Lock() defer h.mu.Unlock() - return append([]string(nil), h.events...), h.openCount, - h.startupSyncs, h.localPushSyncs + return append([]string(nil), h.events...), h.openCount, h.startupSyncs } func (h *pushWatchOwnerHarness) watcherCallback() syncpkg.WatchCallback { @@ -748,9 +791,9 @@ type pushWatchPGTarget struct { func (*pushWatchPGTarget) EnsureSchema(context.Context) error { return nil } func (t *pushWatchPGTarget) PushWithOptions( - _ context.Context, _ storage.PushOptions, _ func(storage.PushProgress), + _ context.Context, opts storage.PushOptions, _ func(storage.PushProgress), ) (storage.PushResult, error) { - attempt, partial := t.harness.nextPGAttempt() + attempt, partial := t.harness.nextLocalAttempt(opts.Full) if partial { return storage.PushResult{Errors: 1}, nil } @@ -762,10 +805,40 @@ func isLocalEnginePushWatchOwner(name string) bool { return name == "local PostgreSQL" || name == "local DuckDB" } +// daemonPushWatchOwnerCases returns the owners that delegate sync to the +// daemon. Their watcher dispatch waits for the startup push, and authoritative +// batches wait for a push acknowledgement; local owners sync on their own lane +// instead (see archive_push_decouple_test.go). +func daemonPushWatchOwnerCases(t *testing.T) []pushWatchOwnerCase { + t.Helper() + var daemon []pushWatchOwnerCase + for _, owner := range pushWatchOwnerCases(t) { + if !isLocalEnginePushWatchOwner(owner.name) { + daemon = append(daemon, owner) + } + } + return daemon +} + +// localPushWatchTestDebounce is shorter than the floor interval so tests can +// tell the local sync retry apart from the periodic sync. +const localPushWatchTestDebounce = time.Minute + +func localPushWatchOwnerCases(t *testing.T) []pushWatchOwnerCase { + t.Helper() + var local []pushWatchOwnerCase + for _, owner := range pushWatchOwnerCases(t) { + if isLocalEnginePushWatchOwner(owner.name) { + local = append(local, owner) + } + } + return local +} + func TestPushWatchProductionOwnersRetainPartialStartupUntilPeriodicSuccess( t *testing.T, ) { - for _, owner := range pushWatchOwnerCases(t) { + for _, owner := range daemonPushWatchOwnerCases(t) { t.Run(owner.name, func(t *testing.T) { synctest.Test(t, func(t *testing.T) { h := newPushWatchOwnerHarness(1) @@ -776,13 +849,11 @@ func TestPushWatchProductionOwnersRetainPartialStartupUntilPeriodicSuccess( first := receiveArchiveTest(t, h.attempts) require.Equal(t, 1, first.index) - if !isLocalEnginePushWatchOwner(owner.name) { - assert.Equal(t, reasonStartup, first.reason) - } + assert.Equal(t, reasonStartup, first.reason) synctest.Wait() pending, waiters := h.pending() - require.True(t, pending && waiters == 1, - "partial startup must retain one dirty generation and waiter") + require.True(t, pending && waiters == 0, + "partial startup must stay queued for retry") select { case <-h.opened: require.Fail(t, "partial startup opened watcher dispatch") @@ -792,29 +863,18 @@ func TestPushWatchProductionOwnersRetainPartialStartupUntilPeriodicSuccess( h.floor <- time.Now() second := receiveArchiveTest(t, h.attempts) require.Equal(t, 2, second.index) - if !isLocalEnginePushWatchOwner(owner.name) { - assert.Equal(t, reasonInterval, second.reason) - } + assert.Equal(t, reasonInterval, second.reason) receiveArchiveTest(t, h.opened) synctest.Wait() pending, waiters = h.pending() require.True(t, !pending && waiters == 0, "complete periodic push must drain startup work") - events, opens, startupSyncs, localSyncs := h.snapshot() + events, opens, _ := h.snapshot() require.NotEmpty(t, events) assert.Equal(t, "collect", events[0], "watcher collection must precede startup work") assert.Equal(t, 1, opens) - if isLocalEnginePushWatchOwner(owner.name) { - assert.Equal(t, 1, startupSyncs) - assert.GreaterOrEqual(t, localSyncs, 2, - "initial and retry pushes each run local sync") - assert.Less(t, indexOfEvent(events, "startup-sync"), - indexOfEvent(events, "push")) - assert.Less(t, indexOfEvent(events, "local-sync"), - indexOfEvent(events, "push")) - } cancel() require.NoError(t, receiveArchiveTest(t, done)) @@ -826,7 +886,7 @@ func TestPushWatchProductionOwnersRetainPartialStartupUntilPeriodicSuccess( func TestPushWatchProductionOwnersRetryPartialAuthoritativeBatch( t *testing.T, ) { - for _, owner := range pushWatchOwnerCases(t) { + for _, owner := range daemonPushWatchOwnerCases(t) { t.Run(owner.name, func(t *testing.T) { synctest.Test(t, func(t *testing.T) { h := newPushWatchOwnerHarness(2) @@ -837,9 +897,7 @@ func TestPushWatchProductionOwnersRetryPartialAuthoritativeBatch( first := receiveArchiveTest(t, h.attempts) require.Equal(t, 1, first.index) - if !isLocalEnginePushWatchOwner(owner.name) { - assert.Equal(t, reasonStartup, first.reason) - } + assert.Equal(t, reasonStartup, first.reason) receiveArchiveTest(t, h.opened) callback := h.watcherCallback() require.NotNil(t, callback) @@ -854,9 +912,7 @@ func TestPushWatchProductionOwnersRetryPartialAuthoritativeBatch( h.floor <- time.Now() second := receiveArchiveTest(t, h.attempts) require.Equal(t, 2, second.index) - if !isLocalEnginePushWatchOwner(owner.name) { - assert.Equal(t, reasonInterval, second.reason) - } + assert.Equal(t, reasonInterval, second.reason) synctest.Wait() select { case err := <-callbackDone: @@ -869,14 +925,12 @@ func TestPushWatchProductionOwnersRetryPartialAuthoritativeBatch( h.floor <- time.Now() third := receiveArchiveTest(t, h.attempts) require.Equal(t, 3, third.index) - if !isLocalEnginePushWatchOwner(owner.name) { - assert.Equal(t, reasonInterval, third.reason) - } + assert.Equal(t, reasonInterval, third.reason) require.NoError(t, receiveArchiveTest(t, callbackDone)) synctest.Wait() pending, waiters = h.pending() require.True(t, !pending && waiters == 0) - _, opens, _, _ := h.snapshot() + _, opens, _ := h.snapshot() assert.Equal(t, 1, opens, "runtime retries must not reopen dispatch") cancel() @@ -1101,7 +1155,7 @@ func TestDaemonPushWatchOwnersSuppressOpenCodeSHMOnlyBatches(t *testing.T) { "repeated opencode.db-shm batches must not mark the push loop dirty") assert.Zero(t, waiters, "suppressed batches must not register an acknowledgement waiter") - events, _, _, _ := h.snapshot() + events, _, _ := h.snapshot() t.Logf("owner=%s path=opencode.db-shm push_pending=%t waiters=%d pushes=%d", owner.name, pending, waiters, countEvents(events, "push")) @@ -1334,26 +1388,6 @@ func TestPushWatchRelevancePreservesAuthoritativeBatches(t *testing.T) { } } -func TestArchivePushWatchCallbackRetainsUnfilteredBatchForPush(t *testing.T) { - root := t.TempDir() - cfg := config.Config{ - AgentDirs: map[parser.AgentType][]string{ - parser.AgentClaude: {root}, - }, - } - path := filepath.Join(root, "session.jsonl") - loop, _, _ := newTestLoop(func(context.Context, pushReason) error { - return nil - }) - callback := archivePushWatchBatchCallback(cfg, loop) - require.NoError(t, callback(t.Context(), syncpkg.WatchBatch{Paths: []string{path}})) - claim, ok := loop.claimPending() - require.True(t, ok) - require.NotNil(t, claim.batch) - assert.Equal(t, []string{path}, claim.batch.Paths) - t.Log("engine_sync=deferred push_scope=changed_batch") -} - func TestResolveArchiveWriteBackendCopiesNoSyncRuntime(t *testing.T) { dataDir := t.TempDir() host, port := testPingServer(t) @@ -1455,13 +1489,14 @@ func TestLocalPGPushWatchGivesDeferredScopesAPollingOwner(t *testing.T) { label: label, }, func() {} }, - replicaStartupSync: func(context.Context, *syncpkg.Engine, bool) (bool, error) { + startupSync: func(context.Context, *syncpkg.Engine, bool) (bool, error) { return false, nil }, newReplicaPusher: func(*syncpkg.Engine) *replicaPusher { return &replicaPusher{ - localSync: func(context.Context) error { return nil }, - connect: func(context.Context) (storage.Pusher, error) { return noopPGTarget{}, nil }, + session: archivepush.ReplicaSession{ + Connect: func(context.Context) (storage.Pusher, error) { return noopPGTarget{}, nil }, + }, } }, newUnwatchedPoller: func( @@ -1570,12 +1605,11 @@ func TestLocalDuckDBPushWatchGivesDeferredScopesAPollingOwner(t *testing.T) { label: label, }, func() {} }, - duckDBStartupSync: func(context.Context, *syncpkg.Engine, bool) (bool, error) { + startupSync: func(context.Context, *syncpkg.Engine, bool) (bool, error) { return false, nil }, newDuckDBPusher: func(*syncpkg.Engine) *duckDBPusher { return &duckDBPusher{ - localSync: func(context.Context) error { return nil }, mirrorPush: func(context.Context, bool) (storage.MirrorPushResult, error) { return storage.MirrorPushResult{}, nil }, @@ -1662,3 +1696,32 @@ func testLocalArchiveWriteBackend(t *testing.T) *localArchiveWriteBackend { database: database, } } + +// A deferred DuckDB push wrote nothing, so a --full watch keeps its request +// for the retry. After a daemon rebuilt the archive it stops asking: the +// mirror pushes in full anyway, and asking again would make the daemon +// rebuild the archive on every retry while the mirror stays locked. +func TestDeferredDuckDBWatchPushKeepsFullUntilARebuild(t *testing.T) { + lane := &archiveSyncLane{label: "test watch", openDispatch: func() {}} + lane.fullPending.Store(true) + var deferred storage.MirrorPushResult + deferred.Diagnostics.Deferred = true + deferred.Diagnostics.DeferredReason = "another push is rebuilding the mirror" + rebuilt := deferred + rebuilt.ArchiveRebuilt = true + var fullArgs []bool + var notStarted []bool + for _, res := range []storage.MirrorPushResult{deferred, deferred, rebuilt, deferred} { + err := lane.push(t.Context(), reasonChange, nil, func( + _ context.Context, reason pushReason, full bool, _ *syncpkg.WatchBatch, + ) error { + fullArgs = append(fullArgs, full) + return completeDuckDBWatchPush(res, reason, full) + }) + _, keep := errors.AsType[storage.PushNotStartedError](err) + require.True(t, err == nil || keep, "err: %v", err) + notStarted = append(notStarted, keep) + } + assert.Equal(t, []bool{true, true, true, false}, fullArgs) + assert.Equal(t, []bool{true, true, false, false}, notStarted) +} diff --git a/cmd/agentsview/daemon_push.go b/cmd/agentsview/daemon_push.go index 03595093e1..fa185a566d 100644 --- a/cmd/agentsview/daemon_push.go +++ b/cmd/agentsview/daemon_push.go @@ -12,6 +12,7 @@ import ( "go.kenn.io/agentsview/internal/apiclient" "go.kenn.io/agentsview/internal/server" + "go.kenn.io/agentsview/internal/storage" ) // daemonPushOperation invokes one generated streaming daemon operation and @@ -112,7 +113,7 @@ func postDaemonPush[T, P any]( } resp, payload, stream, err := operation(ctx, api, &body) if resp == nil { - return zero, err + return zero, storage.PushNotStartedError{Err: err} } if resp.StatusCode != http.StatusOK { msg := payload @@ -124,7 +125,9 @@ func postDaemonPush[T, P any]( fallbackAttempted = true continue } - return zero, errors.New(daemonErrorMessage(resp.StatusCode, msg)) + return zero, daemonPushError( + errors.New(daemonErrorMessage(resp.StatusCode, msg)), msg, false, + ) } defer resp.Body.Close() if strings.HasPrefix( @@ -167,10 +170,13 @@ func daemonRejectsWatchScope(status int, body []byte) bool { } // consumeDaemonPushEvents applies daemon progress and terminal events decoded -// by the generated client's stream. +// by the generated client's stream. A stream that ends without a done or +// error event counts as a push that never started, so a watch keeps its +// --full request, unless the daemon reported that its sync rebuilt the +// archive: every mirror pushes in full after that. func consumeDaemonPushEvents[T, P any](stream *runtime.Stream[[]byte], onProgress func(P)) (T, error) { var result, zero T - var done bool + var done, rebuilt bool var pushErr error defer stream.Close() for stream.Next() { @@ -185,6 +191,11 @@ func consumeDaemonPushEvents[T, P any](stream *runtime.Stream[[]byte], onProgres } done = true case "progress": + var phase struct{ Phase string } + if json.Unmarshal(frame.Data, &phase) == nil && + phase.Phase == storage.PushPhaseRebuilt { + rebuilt = true + } if onProgress != nil { var progress P if err := json.Unmarshal(frame.Data, &progress); err != nil { @@ -195,20 +206,44 @@ func consumeDaemonPushEvents[T, P any](stream *runtime.Stream[[]byte], onProgres default: var apiErr apiclient.APIErrorResponse if err := json.Unmarshal(frame.Data, &apiErr); err == nil && apiErr.ErrorData != "" { - pushErr = errors.New(apiErr.ErrorData) + pushErr = daemonPushError(errors.New(apiErr.ErrorData), frame.Data, true) } else { pushErr = fmt.Errorf("daemon push error: %s", frame.Data) } } } if err := stream.Err(); err != nil { - return zero, err + return zero, startedIf(rebuilt, err) } if pushErr != nil { return zero, pushErr } if !done { - return zero, errors.New("daemon push response missing done event") + return zero, startedIf(rebuilt, errors.New("daemon push response missing done event")) } return result, nil } + +// daemonPushError marks err as a push that never started when the daemon's +// error body sets push_started to false. Older daemons omit the field; then +// startedByDefault decides. A rejected request never started a push, and an +// error streamed after the daemon accepted the push may come from one that +// did. +func daemonPushError(err error, body []byte, startedByDefault bool) error { + var status struct { + PushStarted *bool `json:"push_started"` + } + started := startedByDefault + if json.Unmarshal(body, &status) == nil && status.PushStarted != nil { + started = *status.PushStarted + } + return startedIf(started, err) +} + +// startedIf marks err as a push that never started unless started is set. +func startedIf(started bool, err error) error { + if started { + return err + } + return storage.PushNotStartedError{Err: err} +} diff --git a/cmd/agentsview/daemon_push_test.go b/cmd/agentsview/daemon_push_test.go index 48adfbd43b..62c42d7e7f 100644 --- a/cmd/agentsview/daemon_push_test.go +++ b/cmd/agentsview/daemon_push_test.go @@ -3,6 +3,7 @@ package main import ( "encoding/json/jsontext" "encoding/json/v2" + "errors" "io" "net/http" "net/http/httptest" @@ -59,6 +60,31 @@ func TestParseDaemonPushSSE(t *testing.T) { assert.Contains(t, err.Error(), "missing done event") }) + // A dropped stream keeps a --full request unless the daemon rebuilt the + // archive, after which every mirror pushes in full. + t.Run("stream that ends tells whether the archive was rebuilt", func(t *testing.T) { + waitingEvent := "event: progress\n" + `data: {"Phase":"waiting"}` + "\n\n" + rebuiltEvent := "event: progress\n" + `data: {"Phase":"rebuilt"}` + "\n\n" + for _, tt := range []struct { + name string + events []string + started bool + }{ + {"while waiting", []string{waitingEvent}, false}, + {"during a copy", []string{waitingEvent, progressEvent}, false}, + {"after a rebuild", []string{waitingEvent, rebuiltEvent, progressEvent}, true}, + } { + t.Run(tt.name, func(t *testing.T) { + _, err := consumeDaemonPushEvents[storage.PushResult, storage.PushProgress]( + daemonEventStream(strings.NewReader(stream(tt.events...))), nil, + ) + require.ErrorContains(t, err, "missing done event") + _, notStarted := errors.AsType[storage.PushNotStartedError](err) + assert.Equal(t, tt.started, !notStarted) + }) + } + }) + t.Run("terminal report can exceed ten mebibytes", func(t *testing.T) { type largeReport struct { ProjectMetadata string `json:"project_metadata"` @@ -189,3 +215,52 @@ func TestDaemonPushWatchTransportOmitsScopeForKnownOlderDaemon(t *testing.T) { func daemonEventStream(body io.Reader) *runtime.Stream[[]byte] { return runtime.NewEventStream[[]byte](&http.Response{Body: io.NopCloser(body)}) } + +// A push the daemon refused, never received, or reports as not started +// leaves nothing to resume, so a push watch keeps its --full request for the +// retry. An error from an older daemon that accepted the push may come from a +// push that already started. +func TestPostDaemonPushReportsWhetherThePushStarted(t *testing.T) { + serve := func(handler http.HandlerFunc) string { + ts := httptest.NewServer(handler) + t.Cleanup(ts.Close) + return ts.URL + } + streamError := func(data string) string { + return serve(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("event: error\ndata: " + data + "\n\n")) + }) + } + unreachable := httptest.NewServer(http.NotFoundHandler()) + unreachable.Close() + + for _, tt := range []struct { + name string + url string + started bool + }{ + {"daemon refused", serve(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + }), false}, + {"daemon unreachable", unreachable.URL, false}, + {"older daemon streamed an error", streamError(`{"error":"push: conn reset"}`), true}, + {"pre-push resync failed", streamError( + `{"error":"resync failed","push_started":false}`), false}, + {"copy failed without a stream", serve(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error":"push: conn reset","push_started":true}`)) + }), true}, + } { + t.Run(tt.name, func(t *testing.T) { + _, err := postDaemonPush[storage.PushResult, storage.PushProgress]( + t.Context(), transport{URL: tt.url}, "", replicaPushOperations["pg"], + apiclient.DaemonPushRequest{}, nil, + ) + require.Error(t, err) + _, notStarted := errors.AsType[storage.PushNotStartedError](err) + assert.Equal(t, !tt.started, notStarted) + }) + } +} diff --git a/cmd/agentsview/duckdb.go b/cmd/agentsview/duckdb.go index a580c1cf0b..462d757273 100644 --- a/cmd/agentsview/duckdb.go +++ b/cmd/agentsview/duckdb.go @@ -23,7 +23,6 @@ import ( "go.kenn.io/agentsview/internal/pathutil" "go.kenn.io/agentsview/internal/server" "go.kenn.io/agentsview/internal/storage" - syncpkg "go.kenn.io/agentsview/internal/sync" ) type DuckDBPushConfig struct { @@ -43,65 +42,21 @@ type DuckDBPushConfig struct { Automatic bool } -// duckDBPusher runs a local engine sync then pushes to the DuckDB mirror. -// It mirrors pgPusher's watch-loop shape: interval pushes use SyncAll, which -// never tombstones missed deletions, so deferred scopes rely on the separate -// unwatched-root poller wired by DuckDBPushWatch. +// duckDBPusher pushes the archive to the DuckDB mirror from a watch process +// that owns the archive. Local sync runs separately, so each push sends +// whatever the archive holds when it starts. type duckDBPusher struct { - localSync func(context.Context) error - scopedSync func( - context.Context, syncpkg.WatchBatch, *syncpkg.WatchRecoveryScope, - func() error, - ) error - ensurePricing func(context.Context) error - mirrorPush func(context.Context, bool) (storage.MirrorPushResult, error) + mirrorPush func(context.Context, bool) (storage.MirrorPushResult, error) } func (p *duckDBPusher) push( ctx context.Context, reason pushReason, full bool, ) error { - return p.pushBatch(ctx, reason, full, nil, nil) -} - -func (p *duckDBPusher) pushBatch( - ctx context.Context, - reason pushReason, - full bool, - batch *syncpkg.WatchBatch, - recovery *syncpkg.WatchRecoveryScope, -) error { - push := func() error { return p.pushAfterSync(ctx, reason, full) } - if batch != nil { - if p.scopedSync == nil { - return errors.New("scoped local sync is unavailable") - } - return p.scopedSync(ctx, *batch, recovery, push) - } - if err := p.localSync(ctx); err != nil { - return fmt.Errorf("local sync: %w", err) - } - return push() -} - -func (p *duckDBPusher) pushAfterSync( - ctx context.Context, reason pushReason, full bool, -) error { - if p.ensurePricing != nil { - if err := p.ensurePricing(ctx); err != nil { - if ctxErr := ctx.Err(); ctxErr != nil { - return ctxErr - } - log.Printf("warning: pricing refresh failed: %v", err) - } - } - if err := ctx.Err(); err != nil { - return err - } res, err := p.mirrorPush(ctx, full) if err != nil { return err } - return completeDuckDBWatchPush(res, reason) + return completeDuckDBWatchPush(res, reason, full) } type DuckDBQuackServeConfig struct { diff --git a/cmd/agentsview/pg_watch_loop.go b/cmd/agentsview/pg_watch_loop.go index 5d3035e7cb..6d199e586e 100644 --- a/cmd/agentsview/pg_watch_loop.go +++ b/cmd/agentsview/pg_watch_loop.go @@ -44,6 +44,9 @@ type pushLoop struct { // flushTimeout bounds the final shutdown-flush push. Zero means // no bound (used in tests that inject a fake pusher). flushTimeout time.Duration + // onIdle, when set, is called each time Run starts waiting for work. + // Tests use it to act only once no push is running. + onIdle func() } // queuedPushTask is one unit of work owned by the push loop. Producers enqueue @@ -195,16 +198,15 @@ func (l *pushLoop) run(ctx context.Context, wake <-chan struct{}) { var armed bool var fire <-chan time.Time for { + if l.onIdle != nil { + l.onIdle() + } select { case <-ctx.Done(): // Final best-effort flush with a fresh context so the // push is not immediately cancelled. - flushCtx := context.Background() - if l.flushTimeout > 0 { - var cancel context.CancelFunc - flushCtx, cancel = context.WithTimeout(flushCtx, l.flushTimeout) - defer cancel() - } + flushCtx, cancel := l.flushContext() + defer cancel() l.enqueueShutdownTask() l.runNextTask(flushCtx, true) return @@ -354,3 +356,12 @@ func completePushWaiters(waiters []chan error, err error) { close(waiter) } } + +// flushContext returns the context for work after shutdown starts: fresh, so +// it is not already cancelled, and bounded by flushTimeout when set. +func (l *pushLoop) flushContext() (context.Context, context.CancelFunc) { + if l.flushTimeout > 0 { + return context.WithTimeout(context.Background(), l.flushTimeout) + } + return context.WithCancel(context.Background()) +} diff --git a/cmd/agentsview/pg_watch_test.go b/cmd/agentsview/pg_watch_test.go index 6aef19f949..5754a985f3 100644 --- a/cmd/agentsview/pg_watch_test.go +++ b/cmd/agentsview/pg_watch_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.kenn.io/agentsview/internal/apiclient" + "go.kenn.io/agentsview/internal/archivepush" "go.kenn.io/agentsview/internal/config" "go.kenn.io/agentsview/internal/storage" ) @@ -197,6 +198,58 @@ func TestArchiveWriteBackendPGPushWatchReResolvesDaemon(t *testing.T) { assert.NoFileExists(t, filepath.Join(dataDir, "sessions.db")) } +// TestArchiveWriteBackendPGPushWatchRetriesStartupOnResolvedDaemon checks that +// a failed startup push retries against the daemon the runtime file names now. +// A daemon that restarts on a new address must not leave the watch retrying the +// address it launched with. +func TestArchiveWriteBackendPGPushWatchRetriesStartupOnResolvedDaemon(t *testing.T) { + dataDir := t.TempDir() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + setTestVersion(t, "v1.1.0-2-g123456") + stubStartBackgroundServeForTransport(t, func(context.Context, *config.Config, time.Duration, bool) (*DaemonRuntime, error) { + cancel() + return nil, context.Canceled + }) + var launchPushes int + launch := pushRuntimeServer(t, "/api/v1/push/pg", func( + w http.ResponseWriter, + r *http.Request, + ) { + launchPushes++ + http.Error(w, "daemon restarting", http.StatusServiceUnavailable) + }) + var resolvedPushes int + resolved := pushRuntimeServer(t, "/api/v1/push/pg", func( + w http.ResponseWriter, + r *http.Request, + ) { + resolvedPushes++ + cancel() + writeTestJSON(t, w, storage.PushResult{SessionsPushed: 1}) + }) + host, port := splitTestServerURL(t, resolved.URL) + writeDaemonRuntimeForTest(t, dataDir, host, port, "v1.1.0-3-gabcdef", false) + + backend := newDaemonArchiveWriteBackendForTest( + config.Config{DataDir: dataDir}, launch.URL, + ) + err := backend.ReplicaPushWatch( + ctx, pgReplica{}, + storage.ConfiguredReplica{Target: storage.ReplicaTarget{ + URL: "postgres://user:pass@host/db", + }}, + ReplicaPushConfig{}, + nil, + nil, + time.Millisecond, + time.Hour, + ) + require.NoError(t, err) + assert.Equal(t, 1, launchPushes, "only the first push uses the launch address") + assert.GreaterOrEqual(t, resolvedPushes, 1, "the startup retry reaches the resolved daemon") +} + // fakeTarget is a test double for storage.Pusher. type fakeTarget struct { ensureErr error @@ -227,24 +280,30 @@ type pusherRecorder struct { connects int } -// newTestReplicaPusher builds a replicaPusher whose localSync is a no-op and whose -// connect hands out the supplied targets in order, recording each dial. +// newTestReplicaPusher builds a replicaPusher whose connect hands out the +// supplied targets in order, recording each dial. func newTestReplicaPusher(targets ...*fakeTarget) (*replicaPusher, *pusherRecorder) { rec := &pusherRecorder{} - p := &replicaPusher{ - label: "pg watch", - displayName: "PostgreSQL", - localSync: func(context.Context) error { return nil }, - ensurePricing: func(context.Context) error { return nil }, - connect: func(context.Context) (storage.Pusher, error) { + p := testReplicaPusher(archivepush.ReplicaSession{ + EnsurePricing: func(context.Context) error { return nil }, + Connect: func(context.Context) (storage.Pusher, error) { tgt := targets[rec.connects] rec.connects++ return tgt, nil }, - } + }) return p, rec } +func testReplicaPusher(session archivepush.ReplicaSession) *replicaPusher { + return &replicaPusher{ + label: "pg watch", + displayName: "PostgreSQL", + session: session, + vectors: newVectorReconcile(), + } +} + // TestPGPusherScopesChangeVectorPushes pins the watch scoping policy: only // change-triggered pushes after a clean generation-wide vector // reconciliation scope their vector phase; startup and the interval floor @@ -255,7 +314,7 @@ func TestPGPusherScopesChangeVectorPushes(t *testing.T) { Vectors: storage.VectorPushResult{GenerationID: 1}, }} pusher, _ := newTestReplicaPusher(target) - pusher.vectorReconcileNeeded = true + pusher.vectors.needed = true ctx := t.Context() require.NoError(t, pusher.push(ctx, reasonStartup, false)) @@ -289,7 +348,7 @@ func TestPGPusherScopesChangeVectorPushes(t *testing.T) { func TestPGPusherZeroGenerationKeepsReconcile(t *testing.T) { target := &fakeTarget{} pusher, _ := newTestReplicaPusher(target) - pusher.vectorReconcileNeeded = true + pusher.vectors.needed = true ctx := t.Context() require.NoError(t, pusher.push(ctx, reasonStartup, false)) @@ -317,7 +376,7 @@ func TestPGPusherPushErrorForcesReconcile(t *testing.T) { failing := &fakeTarget{pushErr: errors.New("boom")} recovered := &fakeTarget{} pusher, _ := newTestReplicaPusher(failing, recovered) - pusher.vectorReconcileNeeded = false + pusher.vectors.needed = false ctx := t.Context() require.Error(t, pusher.push(ctx, reasonChange, false)) @@ -337,7 +396,7 @@ func TestPGPusherPushErrorForcesReconcile(t *testing.T) { func TestPGPusherThreadsGenerationID(t *testing.T) { target := &fakeTarget{} pusher, _ := newTestReplicaPusher(target) - pusher.vectorReconcileNeeded = true + pusher.vectors.needed = true ctx := t.Context() // Startup reconciles generation id 1 generation-wide. @@ -375,45 +434,36 @@ func TestPGPusherThreadsGenerationID(t *testing.T) { assert.True(t, target.pushOpts[3].ScopeVectorsToChangedSessions) } -func TestPGPusherEnsuresPricingAfterLocalSyncBeforeConnect(t *testing.T) { +func TestPGPusherEnsuresPricingBeforeConnect(t *testing.T) { var events []string target := &fakeTarget{ onPush: func() { events = append(events, "push") }, } - pusher := &replicaPusher{ - label: "pg watch", displayName: "PostgreSQL", - localSync: func(context.Context) error { - events = append(events, "local sync") - return nil - }, - ensurePricing: func(context.Context) error { + pusher := testReplicaPusher(archivepush.ReplicaSession{ + EnsurePricing: func(context.Context) error { events = append(events, "pricing ensure") return nil }, - connect: func(context.Context) (storage.Pusher, error) { + Connect: func(context.Context) (storage.Pusher, error) { events = append(events, "connect") return target, nil }, - } + }) require.NoError(t, pusher.push( t.Context(), reasonChange, false, )) - assert.Equal(t, []string{ - "local sync", "pricing ensure", "connect", "push", - }, events) + assert.Equal(t, []string{"pricing ensure", "connect", "push"}, events) } func TestPGPusherPricingFailureWarnsAndContinues(t *testing.T) { wantErr := errors.New("catalog unavailable") target := &fakeTarget{} logs := captureLogOutput(t) - pusher := &replicaPusher{ - label: "pg watch", displayName: "PostgreSQL", - localSync: func(context.Context) error { return nil }, - ensurePricing: func(context.Context) error { return wantErr }, - connect: func(context.Context) (storage.Pusher, error) { return target, nil }, - } + pusher := testReplicaPusher(archivepush.ReplicaSession{ + EnsurePricing: func(context.Context) error { return wantErr }, + Connect: func(context.Context) (storage.Pusher, error) { return target, nil }, + }) require.NoError(t, pusher.push( t.Context(), reasonChange, false, @@ -426,19 +476,17 @@ func TestPGPusherPricingFailureWarnsAndContinues(t *testing.T) { func TestPGPusherCanceledPricingStopsBeforeConnect(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) connectCalled := false - pusher := &replicaPusher{ - label: "pg watch", displayName: "PostgreSQL", - localSync: func(context.Context) error { return nil }, - ensurePricing: func(got context.Context) error { + pusher := testReplicaPusher(archivepush.ReplicaSession{ + EnsurePricing: func(got context.Context) error { require.Equal(t, ctx, got) cancel() return got.Err() }, - connect: func(context.Context) (storage.Pusher, error) { + Connect: func(context.Context) (storage.Pusher, error) { connectCalled = true return &fakeTarget{}, nil }, - } + }) err := pusher.push(ctx, reasonChange, false) @@ -458,6 +506,46 @@ func requireReconnectAfterTargetError(t *testing.T, first *fakeTarget) { require.Equal(t, 2, rec.connects, "should reconnect after error") } +// TestPgPusherErrorsNameTheFailedStep checks that a watch push error says +// whether the connection or the copy failed. +func TestPgPusherErrorsNameTheFailedStep(t *testing.T) { + connectErr := errors.New("connection refused") + p := testReplicaPusher(archivepush.ReplicaSession{ + EnsurePricing: func(context.Context) error { return nil }, + Connect: func(context.Context) (storage.Pusher, error) { + return nil, connectErr + }, + }) + err := p.push(t.Context(), reasonChange, false) + require.ErrorIs(t, err, connectErr) + assert.Equal(t, "connect: connection refused", err.Error()) + _, notStarted := errors.AsType[storage.PushNotStartedError](err) + assert.True(t, notStarted, "a connect failure never reached the replica") + + pushErr := errors.New("conn reset") + p, _ = newTestReplicaPusher(&fakeTarget{pushErr: pushErr}) + err = p.push(t.Context(), reasonChange, false) + require.ErrorIs(t, err, pushErr) + assert.Equal(t, "push: conn reset", err.Error()) + _, notStarted = errors.AsType[storage.PushNotStartedError](err) + assert.False(t, notStarted, "a copy failure may have started a full push") +} + +// The watch records the push stage with its own progress hook. A hook the +// caller set must keep firing. +func TestPgPusherKeepsTheCallerStageHook(t *testing.T) { + p, _ := newTestReplicaPusher(&fakeTarget{}) + var stages []archivepush.Stage + p.session.OnStage = func(s archivepush.Stage) func() { + stages = append(stages, s) + return nil + } + require.NoError(t, p.push(t.Context(), reasonChange, false)) + assert.Equal(t, []archivepush.Stage{ + archivepush.StageConnect, archivepush.StageSchema, archivepush.StagePush, + }, stages) +} + func TestPgPusher_ConnectsOnceAndReuses(t *testing.T) { target := &fakeTarget{} p, rec := newTestReplicaPusher(target) @@ -475,31 +563,6 @@ func TestPgPusher_ReconnectsAfterEnsureSchemaError(t *testing.T) { requireReconnectAfterTargetError(t, &fakeTarget{ensureErr: errors.New("schema down")}) } -func TestPgPusher_ConnectErrorSurfaced(t *testing.T) { - p := &replicaPusher{ - label: "pg watch", displayName: "PostgreSQL", - localSync: func(context.Context) error { return nil }, - connect: func(context.Context) (storage.Pusher, error) { - return nil, errors.New("dial timeout") - }, - } - require.Error(t, p.push(t.Context(), reasonChange, false)) -} - -func TestPgPusher_LocalSyncErrorSkipsConnect(t *testing.T) { - connects := 0 - p := &replicaPusher{ - label: "pg watch", displayName: "PostgreSQL", - localSync: func(context.Context) error { return errors.New("disk") }, - connect: func(context.Context) (storage.Pusher, error) { - connects++ - return &fakeTarget{}, nil - }, - } - require.Error(t, p.push(t.Context(), reasonChange, false)) - assert.Equal(t, 0, connects, "connect should not run when local sync fails") -} - func TestPgPusher_LogsPartialPushErrors(t *testing.T) { target := &fakeTarget{ pushResult: storage.PushResult{ @@ -516,8 +579,8 @@ func TestPgPusher_LogsPartialPushErrors(t *testing.T) { got := logs.String() assert.Contains(t, got, "pushed 3 sessions, 9 messages, 2 errors") - assert.Contains(t, got, "2 session(s) failed to push; will retry") assert.Contains(t, got, "change") + assert.Contains(t, got, "2 session(s) failed to push; will retry") } func TestPgPusher_LogsSkippedConflicts(t *testing.T) { @@ -702,7 +765,7 @@ func TestPGPusherFullVectorRepairNarrowsToDeferredSessions(t *testing.T) { } target := &fakeTarget{} pusher, _ := newTestReplicaPusher(target) - pusher.fullVectorsPending = true + pusher.vectors.fullPending = true steps := []storage.PushResult{ {Vectors: storage.VectorPushResult{Skipped: true}}, deferred("s1"), diff --git a/cmd/agentsview/replica_embed_test.go b/cmd/agentsview/replica_embed_test.go index d53a13f07d..8b5f7c0c7a 100644 --- a/cmd/agentsview/replica_embed_test.go +++ b/cmd/agentsview/replica_embed_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/archivepush" "go.kenn.io/agentsview/internal/config" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/dbtest" @@ -321,13 +322,12 @@ func TestReplicaWatchEmbedAdoptsPublishedRecipe(t *testing.T) { embedder, err := newReplicaEmbedder(cfg, replica, embedTarget(), archive) require.NoError(t, err) t.Cleanup(func() { assert.NoError(t, embedder.Close()) }) - pusher := &replicaPusher{ - localSync: func(context.Context) error { return nil }, - connect: func(ctx context.Context) (storage.Pusher, error) { + pusher := testReplicaPusher(archivepush.ReplicaSession{ + Connect: func(ctx context.Context) (storage.Pusher, error) { return replica.NewPusher(ctx, storage.ReplicaTarget{}, archive, storage.PusherOptions{VectorSource: embedder}) }, - } - t.Cleanup(pusher.reset) + }) + t.Cleanup(pusher.session.Close) require.NoError(t, pusher.push(t.Context(), reasonStartup, false)) require.Error(t, embedder.startScheduler(t.Context())) assert.Equal(t, []bool{true}, replica.skipped) @@ -466,7 +466,19 @@ func TestReplicaWatchEmbedKeepsFullVectorRepairAfterStartupFailure(t *testing.T) watchHooks: &archivePushWatchHooks{ newLoop: func(label string, debounce, interval time.Duration, work func(context.Context, pushReason, *syncpkg.WatchBatch) error) (*pushLoop, func()) { loopPush = work - loop, ticker := newPushLoopWithLabel(label, debounce, interval, work) + push := work + if notReady { + // Watcher dispatch opens before the startup push, + // so recovery starts when that push returns. + push = func(c context.Context, r pushReason, batch *syncpkg.WatchBatch) error { + err := work(c, r, batch) + if r == reasonStartup { + recoverPush() + } + return err + } + } + loop, ticker := newPushLoopWithLabel(label, debounce, interval, push) if !notReady { loop.after = func(time.Duration) <-chan time.Time { recoverPush() @@ -475,9 +487,10 @@ func TestReplicaWatchEmbedKeepsFullVectorRepairAfterStartupFailure(t *testing.T) } return loop, ticker.Stop }, - replicaStartupSync: func(_ context.Context, _ *syncpkg.Engine, full bool) (bool, error) { + startupSync: func(ctx context.Context, engine *syncpkg.Engine, full bool) (bool, error) { require.True(t, full) - return true, nil + _, err := runPGWatchStartupSync(ctx, engine, full) + return true, err }, startWatcher: func(config.Config, *syncpkg.Engine, syncpkg.WatchCallback, syncpkg.WatcherOptions) (func(), func(), []string) { recoverPush = func() { @@ -515,9 +528,6 @@ func TestReplicaWatchEmbedKeepsFullVectorRepairAfterStartupFailure(t *testing.T) assert.False(t, replica.options[2].Full) assert.False(t, replica.options[2].FullVectors, "a completed vector phase clears repair") } - if notReady { - return func() {}, recoverPush, nil - } return func() {}, func() {}, nil }, }, @@ -547,19 +557,34 @@ func TestReplicaWatchEmbedPushesSessionsWhileBuildWaits(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() var push func(context.Context, pushReason, *syncpkg.WatchBatch) error + var engine *syncpkg.Engine var shutdownStart time.Time + var afterStartup func() backend := &localArchiveWriteBackend{ appCfg: cfg, database: archive, ensurePricing: func(context.Context, *db.DB) error { return nil }, watchHooks: &archivePushWatchHooks{ newLoop: func(label string, debounce, interval time.Duration, work func(context.Context, pushReason, *syncpkg.WatchBatch) error) (*pushLoop, func()) { push = work - loop, ticker := newPushLoopWithLabel(label, debounce, interval, work) + loop, ticker := newPushLoopWithLabel(label, debounce, interval, + func(c context.Context, r pushReason, batch *syncpkg.WatchBatch) error { + err := work(c, r, batch) + if r == reasonStartup { + afterStartup() + } + return err + }) return loop, ticker.Stop }, - replicaStartupSync: func(context.Context, *syncpkg.Engine, bool) (bool, error) { return false, nil }, - startWatcher: func(config.Config, *syncpkg.Engine, syncpkg.WatchCallback, syncpkg.WatcherOptions) (func(), func(), []string) { - return func() {}, func() { + startupSync: func(ctx context.Context, engine *syncpkg.Engine, full bool) (bool, error) { + _, err := runPGWatchStartupSync(ctx, engine, full) + return false, err + }, + startWatcher: func(_ config.Config, e *syncpkg.Engine, _ syncpkg.WatchCallback, _ syncpkg.WatcherOptions) (func(), func(), []string) { + engine = e + // Watcher dispatch opens before the startup push, so the + // scenario starts when that push returns. + afterStartup = func() { select { case <-started: case <-time.After(10 * time.Second): @@ -569,6 +594,8 @@ func TestReplicaWatchEmbedPushesSessionsWhileBuildWaits(t *testing.T) { AddClaudeUser("2026-01-01T00:00:00Z", "missed watcher event"). String() require.NoError(t, os.WriteFile(filepath.Join(cfg.AgentDirs[parser.AgentClaude][0], "-home-proj0", "interval-session.jsonl"), []byte(content), 0o600)) + // Local sync runs apart from the push, as the sync lane does. + engine.SyncAll(ctx, nil) require.NoError(t, push(ctx, reasonInterval, nil)) session, err := archive.GetSession(ctx, "interval-session") require.NoError(t, err) @@ -582,7 +609,8 @@ func TestReplicaWatchEmbedPushesSessionsWhileBuildWaits(t *testing.T) { assert.Equal(t, []string{"connect", "push", "push", "push"}, replica.events) shutdownStart = time.Now() cancel() - }, nil + } + return func() {}, func() {}, nil }, }, } diff --git a/cmd/agentsview/replica_watch.go b/cmd/agentsview/replica_watch.go index 54e36a2708..9cdc42d99c 100644 --- a/cmd/agentsview/replica_watch.go +++ b/cmd/agentsview/replica_watch.go @@ -14,136 +14,137 @@ import ( "time" "github.com/gofrs/flock" + "go.kenn.io/agentsview/internal/archivepush" "go.kenn.io/agentsview/internal/config" "go.kenn.io/agentsview/internal/storage" - syncpkg "go.kenn.io/agentsview/internal/sync" ) -// replicaPusher runs a local sync then pushes to a replica, lazily -// connecting and reconnecting after errors so a transiently -// unreachable database never crashes the daemon. +// replicaPusher pushes the archive to a replica from a watch process that +// owns the archive. Its session keeps one connection and reconnects after +// errors, so a replica that is unreachable for a while never stops the watch. type replicaPusher struct { // label prefixes log lines, e.g. "pg watch". label string // displayName names the replica product in operator-facing warnings. displayName string - localSync func(context.Context) error - scopedSync func( - context.Context, syncpkg.WatchBatch, *syncpkg.WatchRecoveryScope, - func() error, - ) error - ensurePricing func(context.Context) error - connect func(context.Context) (storage.Pusher, error) - target storage.Pusher - // vectorReconcileNeeded is true until a generation-wide vector - // reconciliation succeeds in this watch process, and again after - // any push error or a vector phase that skipped or deferred work. - // While true, change pushes stay generation-wide so nothing waits - // on the interval floor unnecessarily; while false, change pushes - // scope their vector reads to the changed relational sessions. - vectorReconcileNeeded bool - // fullVectorsPending keeps a full watch's generation-wide vector repair - // until a vector phase runs. vectorRepairSessions then keeps the repair - // for each session that phase deferred until the session's vectors push. - fullVectorsPending bool - vectorRepairSessions map[string]struct{} - // lastReconciledVectorGeneration is the replica generation id of the - // last clean generation-wide reconciliation in this process. A scoped - // push carrying it lets the vector phase promote itself to - // generation-wide when the active generation id differs, so a re-embed - // or a reset/drop-and-recreate (by any machine) never leaves a - // generation partially populated. - lastReconciledVectorGeneration int64 + session archivepush.ReplicaSession + vectors vectorReconcile } -// push performs one local-sync-then-push cycle. On any replica error it -// drops the cached connection so the next call reconnects. +// push copies the archive to the replica. Local sync runs separately, so the +// push sends whatever the archive holds when it starts. func (p *replicaPusher) push( ctx context.Context, reason pushReason, full bool, ) error { - return p.pushBatch(ctx, reason, full, nil, nil) + return p.vectors.push( + p.label, p.displayName, reason, full, + func(opts storage.PushOptions) (storage.PushResult, error) { + // Name the step that failed: a connect error needs a different + // fix than an error partway through the copy. + var stage archivepush.Stage + onStage := p.session.OnStage + defer func() { p.session.OnStage = onStage }() + p.session.OnStage = func(s archivepush.Stage) func() { + stage = s + if onStage != nil { + return onStage(s) + } + return nil + } + res, err := p.session.Push(ctx, opts, nil) + if err == nil { + return res, nil + } + if stage == archivepush.StageConnect || stage == archivepush.StagePush { + err = fmt.Errorf("%s: %w", stage, err) + } + // The replica records a full push only once the copy starts. + if stage != archivepush.StagePush { + err = storage.PushNotStartedError{Err: err} + } + return res, err + }, + ) } -func (p *replicaPusher) pushBatch( - ctx context.Context, - reason pushReason, - full bool, - batch *syncpkg.WatchBatch, - recovery *syncpkg.WatchRecoveryScope, -) error { - push := func() error { return p.pushAfterSync(ctx, reason, full) } - if batch != nil { - if p.scopedSync == nil { - return errors.New("scoped local sync is unavailable") - } - return p.scopedSync(ctx, *batch, recovery, push) - } - if err := p.localSync(ctx); err != nil { - return fmt.Errorf("local sync: %w", err) - } - return push() +// vectorReconcile decides, across the pushes of one watch process, when a +// change push may limit its vector phase to the changed sessions. Both archive +// owners use it: a local owner passes the decision to its pusher, and a daemon +// owner sends it with each push request because the daemon builds a new +// pusher for every request. +type vectorReconcile struct { + // needed is true until a generation-wide vector reconciliation succeeds + // in this watch process, and again after any push error or a vector + // phase that skipped or deferred work. While true, change pushes stay + // generation-wide so nothing waits on the interval floor unnecessarily; + // while false, change pushes scope their vector reads to the changed + // relational sessions. + needed bool + // lastGeneration is the replica generation id of the last clean + // generation-wide reconciliation in this process. A scoped push carrying + // it lets the vector phase promote itself to generation-wide when the + // active generation id differs, so a re-embed or a reset/drop-and-recreate + // (by any machine) never leaves a generation partially populated. + lastGeneration int64 + // fullPending keeps a full watch's generation-wide vector repair until + // a vector phase runs. repairSessions then keeps the repair for each + // session that phase deferred until the session's vectors push. Only a + // local owner sets fullPending; a daemon push request has no field for + // it. + fullPending bool + repairSessions map[string]struct{} } -func (p *replicaPusher) pushAfterSync( - ctx context.Context, reason pushReason, full bool, -) error { - if p.ensurePricing != nil { - if err := p.ensurePricing(ctx); err != nil { - if ctxErr := ctx.Err(); ctxErr != nil { - return ctxErr - } - log.Printf("warning: pricing refresh failed: %v", err) - } - } - if err := ctx.Err(); err != nil { - return err - } - if p.target == nil { - t, err := p.connect(ctx) - if err != nil { - return fmt.Errorf("connect: %w", err) - } - p.target = t - } - if err := p.target.EnsureSchema(ctx); err != nil { - p.reset() - return fmt.Errorf("ensure schema: %w", err) - } - repairPending := p.fullVectorsPending || len(p.vectorRepairSessions) > 0 - scoped := scopedVectorPush(reason, full, p.vectorReconcileNeeded || repairPending) - res, err := p.target.PushWithOptions(ctx, storage.PushOptions{ +// newVectorReconcile starts generation-wide: nothing has been reconciled yet. +func newVectorReconcile() vectorReconcile { + return vectorReconcile{needed: true} +} + +// options returns the push options for the next push. A pending vector repair +// keeps the vector phase generation-wide. +func (v *vectorReconcile) options(reason pushReason, full bool) storage.PushOptions { + repairPending := v.fullPending || len(v.repairSessions) > 0 + return storage.PushOptions{ Full: full, - FullVectors: p.fullVectorsPending, - FullVectorSessions: slices.Sorted(maps.Keys(p.vectorRepairSessions)), - ScopeVectorsToChangedSessions: scoped, - LastReconciledVectorGeneration: p.lastReconciledVectorGeneration, - }, nil) + FullVectors: v.fullPending, + FullVectorSessions: slices.Sorted(maps.Keys(v.repairSessions)), + ScopeVectorsToChangedSessions: scopedVectorPush(reason, full, v.needed || repairPending), + LastReconciledVectorGeneration: v.lastGeneration, + } +} + +// record updates the state from the outcome of a push sent with opts. +func (v *vectorReconcile) record( + opts storage.PushOptions, res storage.PushResult, err error, +) { if err != nil { - p.vectorReconcileNeeded = true - p.reset() - return fmt.Errorf("push: %w", err) + v.needed = true + return } if !res.Vectors.Skipped { - p.vectorRepairSessions = nextVectorRepair( - p.fullVectorsPending, p.vectorRepairSessions, - res.Vectors.DeferredSessionIDs, + v.repairSessions = nextVectorRepair( + v.fullPending, v.repairSessions, res.Vectors.DeferredSessionIDs, ) - p.fullVectorsPending = false + v.fullPending = false } - p.vectorReconcileNeeded, p.lastReconciledVectorGeneration = nextVectorReconcile( - p.vectorReconcileNeeded, - p.lastReconciledVectorGeneration, scoped, res, + v.needed, v.lastGeneration = nextVectorReconcile( + v.needed, v.lastGeneration, opts.ScopeVectorsToChangedSessions, res, ) - if res.Errors > 0 { - logReplicaWatchPushResult(p.label, p.displayName, res, reason) - log.Printf( - "%s: %d session(s) failed to push; will retry", - p.label, res.Errors, - ) - return fmt.Errorf("%d session(s) failed to push", res.Errors) +} + +// push runs one watch push to a replica. send performs the push with the +// options chosen here; the two archive owners differ only in how they send it. +func (v *vectorReconcile) push( + label, displayName string, reason pushReason, full bool, + send func(storage.PushOptions) (storage.PushResult, error), +) error { + opts := v.options(reason, full) + res, err := send(opts) + v.record(opts, res, err) + if err != nil { + return err } - logReplicaWatchPushResult(p.label, p.displayName, res, reason) - return nil + return completeReplicaWatchPush(label, displayName, res, reason) } // scopedVectorPush reports whether a push may scope its vector phase @@ -238,13 +239,6 @@ func logReplicaWatchPushResult( ) } -func (p *replicaPusher) reset() { - if p.target != nil { - _ = p.target.Close() - p.target = nil - } -} - // resolveWatchTarget validates the replica config and resolves the project // filters for a watch run. func resolveWatchTarget( diff --git a/docs/pg-sync.md b/docs/pg-sync.md index 929fde13d3..bfa6d0207c 100644 --- a/docs/pg-sync.md +++ b/docs/pg-sync.md @@ -167,6 +167,14 @@ A full push that reaches the end with session errors resumes only while something else still requires a full push. Otherwise the next push is incremental and retries the failed sessions. +After an archive rebuild, the next push sends all sessions again. Once every +session publishes successfully, a full push also removes PostgreSQL sessions +that this archive owns but no longer contains. Project filters limit this +cleanup to the selected projects. Sessions owned by another archive, or whose +owner is unknown, remain untouched. An interrupted push resumes before cleanup. +Codex thread trash state remains available for restoring or permanently deleting +its pages. + If any sessions fail to push, the watermark is not advanced so they are retried on the next run. The exit code is 1 when any errors occur, 0 otherwise. diff --git a/internal/archivepush/archivepush.go b/internal/archivepush/archivepush.go new file mode 100644 index 0000000000..f36c898d12 --- /dev/null +++ b/internal/archivepush/archivepush.go @@ -0,0 +1,146 @@ +// Package archivepush runs one push from the SQLite archive to a replica or +// mirror. The process that owns the archive calls it: the daemon's push routes +// and a CLI push that opened the archive itself. Both run the same steps, so a +// push behaves the same whichever process runs it. +// +// The caller syncs the archive and settles the sync engine first (see +// sync.Engine.SettleForPush). A push does not hold the sync lock, so local +// ingestion keeps writing while it runs; each push takes its cutoff under the +// archive write lock, and later pushes pick up writes that land meanwhile. +package archivepush + +import ( + "context" + "fmt" + "log" + + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/storage" + syncpkg "go.kenn.io/agentsview/internal/sync" +) + +// RefreshPricing refreshes model pricing before a push. A failed refresh is +// logged and the push continues with the prices on hand, unless ctx ended. +func RefreshPricing( + ctx context.Context, refresh func(context.Context) error, +) error { + if refresh != nil { + if err := refresh(ctx); err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + log.Printf("warning: pricing refresh failed: %v", err) + } + } + return ctx.Err() +} + +// Stage names a step of a replica push, for progress output. +type Stage string + +const ( + StageConnect Stage = "connect" + StageSchema Stage = "schema" + StagePush Stage = "push" +) + +// ReplicaSession pushes the archive to one replica. It keeps the connection +// between pushes and drops it after any replica error, so the next push +// reconnects. +type ReplicaSession struct { + // Connect opens the replica pusher. + Connect func(context.Context) (storage.Pusher, error) + // EnsurePricing refreshes model pricing before each push. Optional. + EnsurePricing func(context.Context) error + // OnStage, when set, is called as each stage starts. The returned func, + // when not nil, is called when the stage succeeds. + OnStage func(Stage) func() + // FrictionReview, when set, reviews stale friction findings under this + // engine's sync lock before each push, so pushed sessions carry current + // findings. Set it when the replica stores friction. Optional. + FrictionReview *syncpkg.Engine + + target storage.Pusher +} + +// Push reviews friction, refreshes pricing, connects if needed, prepares the +// schema, and pushes. On a push error the result holds what was copied. +func (s *ReplicaSession) Push( + ctx context.Context, + opts storage.PushOptions, + onProgress func(storage.PushProgress), +) (storage.PushResult, error) { + if engine := s.FrictionReview; engine != nil { + // The push itself runs without the sync lock. + if err := engine.RunExclusive(func() error { + return engine.DrainStaleFrictionLocked(ctx) + }); err != nil { + return storage.PushResult{}, err + } + } + if err := RefreshPricing(ctx, s.EnsurePricing); err != nil { + return storage.PushResult{}, err + } + if s.target == nil { + done := s.stage(StageConnect) + target, err := s.Connect(ctx) + if err != nil { + return storage.PushResult{}, err + } + s.target = target + done() + } + done := s.stage(StageSchema) + if err := s.target.EnsureSchema(ctx); err != nil { + s.Close() + return storage.PushResult{}, fmt.Errorf("schema: %w", err) + } + done() + s.stage(StagePush) + // A failed push still returns what it copied: a vector build can fail + // after the sessions land. + result, err := s.target.PushWithOptions(ctx, opts, onProgress) + if err != nil { + s.Close() + } + return result, err +} + +// Close drops the replica connection. The session can push again afterward. +func (s *ReplicaSession) Close() { + if s.target != nil { + _ = s.target.Close() + s.target = nil + } +} + +func (s *ReplicaSession) stage(stage Stage) func() { + if s.OnStage != nil { + if done := s.OnStage(stage); done != nil { + return done + } + } + return func() {} +} + +// MirrorPush pushes the archive to a local mirror such as DuckDB. +type MirrorPush struct { + Mirror storage.Mirror + Config config.DuckDBConfig + Archive *db.DB + Options storage.MirrorPushOptions + // EnsurePricing refreshes model pricing before each push. Optional. + EnsurePricing func(context.Context) error +} + +// Push refreshes pricing and pushes. full rewrites the whole mirror. +func (m MirrorPush) Push( + ctx context.Context, full bool, + onProgress func(storage.MirrorPushProgress), +) (storage.MirrorPushResult, error) { + if err := RefreshPricing(ctx, m.EnsurePricing); err != nil { + return storage.MirrorPushResult{}, err + } + return m.Mirror.Push(ctx, m.Config, m.Archive, m.Options, full, onProgress) +} diff --git a/internal/clickhouse/push.go b/internal/clickhouse/push.go index 7cef6b4752..9cb3ea9ca7 100644 --- a/internal/clickhouse/push.go +++ b/internal/clickhouse/push.go @@ -43,6 +43,19 @@ func newPushVersion() uint64 { // session succeeded. func (s *Sync) PushWithOptions( ctx context.Context, opts storage.PushOptions, onProgress func(storage.PushProgress), +) (storage.PushResult, error) { + s.fullRunRecorded = false + result, err := s.pushWithOptions(ctx, opts, onProgress) + if err != nil && opts.Full && !s.fullRunRecorded { + // The push failed before it recorded the requested full push, so + // nothing can resume it. + err = storage.PushNotStartedError{Err: err} + } + return result, err +} + +func (s *Sync) pushWithOptions( + ctx context.Context, opts storage.PushOptions, onProgress func(storage.PushProgress), ) (storage.PushResult, error) { start := time.Now() var result storage.PushResult @@ -57,7 +70,10 @@ func (s *Sync) PushWithOptions( onProgress(storage.PushProgress{Phase: "preparing"}) } version := newPushVersion() - cutoff := time.Now().UTC().Format(localSyncTimestampLayout) + cutoff, err := s.local.MirrorPushCutoff(ctx) + if err != nil { + return result, err + } meta, err := readMetadata(ctx, conn, s.archiveKey(lastPushCutoffKeyBase), @@ -191,6 +207,7 @@ func (s *Sync) PushWithOptions( if err := writeMetadata(ctx, conn, map[string]string{s.archiveKey(fullPushRunKeyBase): encoded}); err != nil { return result, err } + s.fullRunRecorded = true } if !full { if err := s.applyDeletionDelta(ctx, storedDeletion, localDeletion, &result); err != nil { diff --git a/internal/clickhouse/push_not_started_chtest_test.go b/internal/clickhouse/push_not_started_chtest_test.go new file mode 100644 index 0000000000..518804d9ef --- /dev/null +++ b/internal/clickhouse/push_not_started_chtest_test.go @@ -0,0 +1,42 @@ +//go:build chtest + +package clickhouse + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/storage" +) + +// A --full push that fails before it records its full push reports that it +// never started, so a watch keeps the --full request. One that fails after it +// recorded the run resumes that run instead. +func TestFullPushReportsWhetherItRecordedItsRun(t *testing.T) { + ctx := t.Context() + local, target := seedFixture(t) + canceled, cancel := context.WithCancel(ctx) + cancel() + _, err := newTestSync(t, local, target, storage.PusherOptions{}). + PushWithOptions(canceled, storage.PushOptions{Full: true}, nil) + require.ErrorIs(t, err, context.Canceled) + _, notStarted := errors.AsType[storage.PushNotStartedError](err) + assert.True(t, notStarted, "the push failed before it recorded a run") + + s := newTestSync(t, local, target, storage.PusherOptions{}) + killCtx, kill := context.WithCancel(ctx) + defer kill() + s.hooks = &pushHooks{beforeSessionRows: func([]db.Session) error { + kill() + return killCtx.Err() + }} + _, err = s.PushWithOptions(killCtx, storage.PushOptions{Full: true}, nil) + require.ErrorIs(t, err, context.Canceled) + _, notStarted = errors.AsType[storage.PushNotStartedError](err) + assert.False(t, notStarted, "the push recorded a run the next push resumes") +} diff --git a/internal/clickhouse/sync.go b/internal/clickhouse/sync.go index 9d5290c70f..9fe8a82aaa 100644 --- a/internal/clickhouse/sync.go +++ b/internal/clickhouse/sync.go @@ -13,10 +13,6 @@ import ( "go.kenn.io/agentsview/internal/storage" ) -// localSyncTimestampLayout is the layout ListSessionsForMirrorWindow -// compares sync_marker against. -const localSyncTimestampLayout = "2006-01-02T15:04:05.000Z" - // Sync pushes the SQLite archive into one ClickHouse mirror. Push is // one-way: it never reads content back into SQLite. type Sync struct { @@ -44,6 +40,10 @@ type Sync struct { // snapshot syncUsagePrices loaded. pricer *usagePricer + // fullRunRecorded reports that the current push recorded its full push + // run, so a later failure can resume it. + fullRunRecorded bool + // unownedPruned records that this Sync already removed this machine's // mirror sessions without an archive id that the archive no longer // holds. Pushes always record their archive, so no new ones appear. diff --git a/internal/db/db.go b/internal/db/db.go index 8132145036..f7b04b9eb2 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -5629,6 +5629,38 @@ func (db *DB) SetSyncState(ctx context.Context, key, value string) error { return err } +// ErrDatabaseReplaced reports that the archive was rebuilt since the caller +// read its database ID. +var ErrDatabaseReplaced = errors.New("the local archive was rebuilt") + +// SetSyncStatesForDatabase writes values to the pg_sync_state table in one +// transaction, and only while the archive's database ID is still databaseID. +// An archive swap cannot land between the writes, and a caller whose archive +// was rebuilt gets ErrDatabaseReplaced instead of writing into the +// replacement. +func (db *DB) SetSyncStatesForDatabase( + ctx context.Context, databaseID string, values map[string]string, +) error { + return db.Update(ctx, func(tx *sql.Tx) error { + var current string + if err := tx.QueryRowContext(ctx, + `SELECT value FROM archive_metadata WHERE key = ?`, + archiveMetadataDatabaseIDKey, + ).Scan(¤t); err != nil { + return fmt.Errorf("reading database id: %w", err) + } + if strings.TrimSpace(current) != databaseID { + return ErrDatabaseReplaced + } + for key, value := range values { + if _, err := tx.ExecContext(ctx, setSyncStateQuery, key, value); err != nil { + return fmt.Errorf("writing sync state %s: %w", key, err) + } + } + return nil + }) +} + // DeleteSyncStateByPrefix removes every pg_sync_state row whose key starts // with prefix. Used to clean up state left behind by superseded sync // designs (e.g. the pre-schema-v3 DuckDB push watermarks, now tracked in diff --git a/internal/db/db_test.go b/internal/db/db_test.go index 8d2d57ef82..1ace90f61e 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -6952,6 +6952,10 @@ func TestCopySyncStateFrom_OnlyCopiesDurableKeys(t *testing.T) { "seed source marker") require.NoError(t, srcDB.SetSyncState(t.Context(), "artifact_origin_id", "laptop-a1b2c3"), "seed source artifact origin") + require.NoError(t, srcDB.SetSyncState(t.Context(), "pg_push_state_saved_v1:scope", "1"), + "seed source scoped push-state mark") + require.NoError(t, srcDB.SetSyncState(t.Context(), "last_push_at:scope", "old-push"), + "seed source scoped push watermark") require.NoError(t, srcDB.SetSyncState(t.Context(), "last_sync_started_at", "old-start"), "seed source started") require.NoError(t, srcDB.SetSyncState(t.Context(), "last_sync_finished_at", "old-finish"), @@ -6981,6 +6985,15 @@ func TestCopySyncStateFrom_OnlyCopiesDurableKeys(t *testing.T) { require.NoError(t, err, "GetSyncState pg_push_marker_id") assert.Equal(t, "marker-123", gotMarker) + // The mark survives and the watermark does not, so the next push of + // that scope knows it lost its push state. + gotSaved, err := dstDB.GetSyncState(t.Context(), "pg_push_state_saved_v1:scope") + require.NoError(t, err, "GetSyncState pg_push_state_saved_v1:scope") + assert.Equal(t, "1", gotSaved) + gotPush, err := dstDB.GetSyncState(t.Context(), "last_push_at:scope") + require.NoError(t, err, "GetSyncState last_push_at:scope") + assert.Empty(t, gotPush) + gotOrigin, err := dstDB.GetSyncState(t.Context(), "artifact_origin_id") require.NoError(t, err, "GetSyncState artifact_origin_id") assert.Equal(t, "laptop-a1b2c3", gotOrigin, diff --git a/internal/db/orphaned.go b/internal/db/orphaned.go index 5dea879e54..0cd681beb6 100644 --- a/internal/db/orphaned.go +++ b/internal/db/orphaned.go @@ -478,6 +478,8 @@ func (d *DB) CopySyncStateFrom(sourcePath string) error { WHERE key = 'pg_push_marker_id' OR key = 'pg_full_push_run_v1' OR key LIKE 'pg\_full\_push\_run\_v1:%' ESCAPE '\' + OR key = 'pg_push_state_saved_v1' + OR key LIKE 'pg\_push\_state\_saved\_v1:%' ESCAPE '\' OR key LIKE 'artifact\_%' ESCAPE '\' OR key LIKE 'machine\_label:%' ESCAPE '\' OR key LIKE 'machine\_alias:%' ESCAPE '\' diff --git a/internal/db/sessions.go b/internal/db/sessions.go index 1e0045f69c..8441b199a4 100644 --- a/internal/db/sessions.go +++ b/internal/db/sessions.go @@ -10,6 +10,7 @@ import ( "encoding/json/v2" "errors" "fmt" + "log" pathpkg "path" "path/filepath" "sort" @@ -6796,6 +6797,77 @@ func (db *DB) CountSessionsForMirrorScope( return count, nil } +// MirrorPushCutoff returns the cutoff a mirror push records as its next +// window's lower bound. It reads the clock while holding the write lock, so +// no write transaction is open: every write stamped before the cutoff has +// committed and is visible to the push's later listing, and every later write +// is stamped at or after the cutoff. A cutoff read without the lock can fall +// after the stamp of a write that commits only after the push lists sessions; +// no later window would list that write. +// +// It also waits while the writer is closed for a worker handoff. A sync-worker +// child writes the archive then, outside this process's write lock, so a +// cutoff taken during the handoff has the same gap. MirrorPushCutoff returns +// ctx.Err() if ctx ends first. +func (db *DB) MirrorPushCutoff(ctx context.Context) (string, error) { + for { + if err := db.WaitForWriter(ctx); err != nil { + return "", err + } + db.mu.Lock() + if !db.writerClosed.Load() { + cutoff := time.Now().UTC().Format(mirrorWindowBoundLayout) + db.mu.Unlock() + return cutoff, nil + } + db.mu.Unlock() + } +} + +// WaitForWriter blocks while the writer is closed for a worker handoff. It +// returns ctx.Err() if ctx ends first. A write can still fail if a new +// handoff starts after it returns. A wait that lasts writerHandoffNoticeAfter +// is logged and reported to the notice set by WithWriterWaitNotice. +func (db *DB) WaitForWriter(ctx context.Context) error { + start := time.Now() + announced := false + for db.writerClosed.Load() { + if !announced && time.Since(start) >= writerHandoffNoticeAfter { + announced = true + log.Printf("waiting for a local sync to finish before writing the archive") + if notice, ok := ctx.Value(writerWaitNoticeKey{}).(func()); ok { + notice() + } + } + select { + case <-ctx.Done(): + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return fmt.Errorf("gave up waiting for the local sync to finish: %w", ctx.Err()) + } + return fmt.Errorf("stopped while waiting for the local sync to finish: %w", ctx.Err()) + case <-time.After(writerHandoffPollInterval): + } + } + return nil +} + +type writerWaitNoticeKey struct{} + +// WithWriterWaitNotice returns a context whose WaitForWriter calls notice when +// a wait for a worker handoff lasts long enough to log. A daemon push uses it +// to tell its client why the push paused. +func WithWriterWaitNotice(ctx context.Context, notice func()) context.Context { + return context.WithValue(ctx, writerWaitNoticeKey{}, notice) +} + +// writerHandoffPollInterval spaces checks for the end of a worker handoff. +// Handoffs last seconds to minutes. +const writerHandoffPollInterval = 50 * time.Millisecond + +// writerHandoffNoticeAfter is how long WaitForWriter waits before it logs that +// it is waiting, so a short handoff stays quiet. +const writerHandoffNoticeAfter = 5 * time.Second + // normalizeMirrorWindowBound parses an RFC3339Nano timestamp and formats it // as ms-precision UTC text matching the sync_marker column format, so the // bound compares correctly against trigger-maintained markers. @@ -6804,9 +6876,12 @@ func normalizeMirrorWindowBound(bound string) (string, error) { if err != nil { return "", fmt.Errorf("parsing mirror window bound %q: %w", bound, err) } - return parsed.UTC().Format("2006-01-02T15:04:05.000Z"), nil + return parsed.UTC().Format(mirrorWindowBoundLayout), nil } +// mirrorWindowBoundLayout matches the trigger-maintained sync_marker text. +const mirrorWindowBoundLayout = "2006-01-02T15:04:05.000Z" + // SessionProjectsByIDs returns each session's current project keyed by session // ID. IDs with no sessions row are absent from the result, so a caller can tell // "unknown session" (missing key) from "empty project" (present, empty value). diff --git a/internal/db/sessions_sync_marker_test.go b/internal/db/sessions_sync_marker_test.go index 7f31b86645..14971453cb 100644 --- a/internal/db/sessions_sync_marker_test.go +++ b/internal/db/sessions_sync_marker_test.go @@ -3,11 +3,19 @@ package db import ( + "context" + "database/sql" "encoding/json" + "runtime" + "sync/atomic" "testing" + "testing/synctest" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/testutil" ) type usageVersion struct { @@ -475,3 +483,187 @@ func TestSyncMarkerMalformedCreatedAtIsDropped(t *testing.T) { assert.Empty(t, readMarker("sm-malformed")) assert.Equal(t, "2026-07-02T09:30:00.000Z", readMarker("sm-mixed")) } + +// A push that starts while a sync transaction is open must list that +// transaction's session. Otherwise the session falls below every later +// incremental window, whose cutoff is already past its stamp. +func TestMirrorPushCutoffCoversOpenWriteTransaction(t *testing.T) { + database := testDB(t) + ctx := t.Context() + insertSession(t, database, "written", "proj") + _, err := database.getWriter().ExecContext(ctx, + `UPDATE sessions SET created_at = '2026-07-01T10:00:00.000Z', + local_modified_at = '2026-07-01T10:00:00.000Z' WHERE id = 'written'`) + require.NoError(t, err) + since := "2026-07-02T00:00:00Z" + // A stamp taken just before the push reads its cutoff. + stamp := time.Now().Add(-time.Minute).UTC().Format("2006-01-02T15:04:05.000Z") + + stamped := make(chan struct{}) + release := make(chan struct{}) + committed := make(chan error, 1) + go func() { + committed <- database.Update(ctx, func(tx *sql.Tx) error { + if _, err := tx.ExecContext(ctx, + `UPDATE sessions SET local_modified_at = ? WHERE id = 'written'`, stamp, + ); err != nil { + return err + } + close(stamped) + <-release + return nil + }) + }() + <-stamped + + type listing struct { + ids []string + err error + } + listed := make(chan listing, 1) + go func() { + if _, err := database.MirrorPushCutoff(ctx); err != nil { + listed <- listing{err: err} + return + } + sessions, err := database.ListSessionsForMirrorWindow(ctx, since, nil, nil) + ids := make([]string, 0, len(sessions)) + for _, s := range sessions { + ids = append(ids, s.ID) + } + listed <- listing{ids, err} + }() + // Commit only once the push has listed or is waiting for the write + // lock, so a push that ignores the open transaction lists first. + var got listing + received := false + for !received && !testutil.GoroutineBlockedOnMutexIn("(*DB).MirrorPushCutoff") { + select { + case got = <-listed: + received = true + default: + runtime.Gosched() + } + } + close(release) + if !received { + got = <-listed + } + require.NoError(t, <-committed) + require.NoError(t, got.err) + + assert.Contains(t, got.ids, "written", + "the push must list the transaction that was open when it started") +} + +// A sync-worker child writes the archive while this process has its writer +// closed. A push that starts then must list the child's open transaction once +// it commits, so the cutoff waits for the writer to reopen. +func TestMirrorPushCutoffWaitsForWorkerHandoff(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + database := testDB(t) + ctx := t.Context() + insertSession(t, database, "child-written", "proj") + _, err := database.getWriter().ExecContext(ctx, + `UPDATE sessions SET created_at = '2026-07-01T10:00:00.000Z', + local_modified_at = '2026-07-01T10:00:00.000Z' WHERE id = 'child-written'`) + require.NoError(t, err) + since := "2026-07-02T00:00:00Z" + stamp := "2026-07-03T00:00:00.000Z" + + require.NoError(t, database.CloseWriter()) + child, err := sql.Open(sqliteArchiveDriverName, makeDSN(database.path, false)) + require.NoError(t, err) + defer child.Close() + tx, err := child.BeginTx(ctx, nil) + require.NoError(t, err) + _, err = tx.ExecContext(ctx, + `UPDATE sessions SET local_modified_at = ? WHERE id = 'child-written'`, stamp) + require.NoError(t, err) + + type listing struct { + ids []string + err error + } + listed := make(chan listing, 1) + go func() { + if _, err := database.MirrorPushCutoff(ctx); err != nil { + listed <- listing{err: err} + return + } + sessions, err := database.ListSessionsForMirrorWindow(ctx, since, nil, nil) + ids := make([]string, 0, len(sessions)) + for _, s := range sessions { + ids = append(ids, s.ID) + } + listed <- listing{ids, err} + }() + // A push that ignores the handoff lists before the child commits. + synctest.Wait() + var got listing + received := false + select { + case got = <-listed: + received = true + default: + } + require.NoError(t, tx.Commit()) + require.NoError(t, child.Close()) + require.NoError(t, database.ReopenWriter()) + if !received { + got = <-listed + } + require.NoError(t, got.err) + assert.Contains(t, got.ids, "child-written", + "the push must list the worker write that was open when it started") + }) +} + +func TestMirrorPushCutoffStopsWaitingWhenContextEnds(t *testing.T) { + database := testDB(t) + require.NoError(t, database.CloseWriter()) + t.Cleanup(func() { require.NoError(t, database.ReopenWriter()) }) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err := database.MirrorPushCutoff(ctx) + require.ErrorIs(t, err, context.Canceled) +} + +// A daemon push tells its client when it waits on a worker handoff long enough +// to log it. A short handoff stays quiet. +func TestWaitForWriterReportsALongWait(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + database := testDB(t) + require.NoError(t, database.CloseWriter()) + var notices atomic.Int32 + ctx := WithWriterWaitNotice(t.Context(), func() { notices.Add(1) }) + done := make(chan error, 1) + go func() { done <- database.WaitForWriter(ctx) }() + + time.Sleep(writerHandoffNoticeAfter - time.Second) + synctest.Wait() + assert.Zero(t, notices.Load(), "a short wait must stay quiet") + time.Sleep(2 * writerHandoffNoticeAfter) + synctest.Wait() + assert.Equal(t, int32(1), notices.Load()) + + require.NoError(t, database.ReopenWriter()) + require.NoError(t, <-done) + }) +} + +// A push that runs out of time while a local sync holds the writer says it gave +// up waiting, so the reader knows the push timed out. +func TestWaitForWriterSaysItGaveUpOnATimeout(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + database := testDB(t) + require.NoError(t, database.CloseWriter()) + t.Cleanup(func() { require.NoError(t, database.ReopenWriter()) }) + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + err := database.WaitForWriter(ctx) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.EqualError(t, err, + "gave up waiting for the local sync to finish: context deadline exceeded") + }) +} diff --git a/internal/duckdb/rebuild.go b/internal/duckdb/rebuild.go index 00b02b1486..8cb234894a 100644 --- a/internal/duckdb/rebuild.go +++ b/internal/duckdb/rebuild.go @@ -308,9 +308,9 @@ func buildMirror( s, err := New(ctx, file, local, machine, opts) if err != nil { if isMirrorHeldError(err) { - return storage.MirrorPushResult{}, rebuildInProgressError(path, err) + err = rebuildInProgressError(path, err) } - return storage.MirrorPushResult{}, err + return storage.MirrorPushResult{}, storage.PushNotStartedError{Err: err} } result, buildErr := buildMirrorInto(ctx, s, opts, run, onProgress) if closeErr := s.Close(); closeErr != nil && buildErr == nil { @@ -446,9 +446,9 @@ type rebuildSnapshot struct { cutoff string deletionRevision int64 // sourceDatabaseID identifies the archive generation this rebuild reads - // from (see mirrorMetadata.SourceDatabaseID). It cannot change while the - // rebuild's *db.DB handle is open, so capturing it here is a convenience - // rather than a race guard. + // from (see mirrorMetadata.SourceDatabaseID). The daemon can replace the + // archive while the rebuild runs; recording the id read before the + // listing makes the next push rebuild again then. sourceDatabaseID string // sourceArchiveID is the stable provenance identity stamped onto every // mirrored row. A repair can change it without changing database_id. @@ -459,6 +459,10 @@ type rebuildSnapshot struct { // post-rebuild mirror metadata. It must be called before the rebuild lists // sessions to push; see rebuildSnapshot for why. func captureRebuildSnapshot(ctx context.Context, local *db.DB) (rebuildSnapshot, error) { + cutoff, err := local.MirrorPushCutoff(ctx) + if err != nil { + return rebuildSnapshot{}, err + } deletionRevision, err := local.SessionDeletionPublicationRevision(ctx) if err != nil { return rebuildSnapshot{}, err @@ -476,7 +480,7 @@ func captureRebuildSnapshot(ctx context.Context, local *db.DB) (rebuildSnapshot, ) } return rebuildSnapshot{ - cutoff: time.Now().UTC().Format(localSyncTimestampLayout), + cutoff: cutoff, deletionRevision: deletionRevision, sourceDatabaseID: sourceDatabaseID, sourceArchiveID: sourceArchiveID, @@ -497,15 +501,19 @@ func buildMirrorInto( onProgress func(storage.MirrorPushProgress), ) (storage.MirrorPushResult, error) { start := time.Now() + // Until the run is recorded in the rebuild file, no later push can resume + // this rebuild. if err := createSchema(ctx, s.duck); err != nil { - return storage.MirrorPushResult{}, err + return storage.MirrorPushResult{}, storage.PushNotStartedError{Err: err} } encodedRun, err := json.Marshal(run) if err != nil { - return storage.MirrorPushResult{}, fmt.Errorf("encoding duckdb rebuild run: %w", err) + return storage.MirrorPushResult{}, storage.PushNotStartedError{ + Err: fmt.Errorf("encoding duckdb rebuild run: %w", err), + } } if err := recordMetadataKey(ctx, s.duck, rebuildRunMetadataKey, string(encodedRun)); err != nil { - return storage.MirrorPushResult{}, err + return storage.MirrorPushResult{}, storage.PushNotStartedError{Err: err} } snapshot, err := captureRebuildSnapshot(ctx, s.local) if err != nil { diff --git a/internal/duckdb/rebuild_test.go b/internal/duckdb/rebuild_test.go index 1790649b10..43e8d8561e 100644 --- a/internal/duckdb/rebuild_test.go +++ b/internal/duckdb/rebuild_test.go @@ -710,3 +710,23 @@ func TestSweepStaleTempFilesHandlesGlobMetacharactersInDirectory(t *testing.T) { assert.NoFileExists(t, staleTmp, "a stale temp file in a glob-metacharacter directory must still be swept") } + +// A --full push that cannot start its rebuild, because another push holds +// the rebuild lock, reports that it never started, so a watch keeps the +// --full request for its retry. +func TestFullPushBlockedByAnotherRebuildIsNotStarted(t *testing.T) { + ctx := t.Context() + local := newLocalDB(t) + seedRebuildFixture(t, local) + path := filepath.Join(t.TempDir(), "mirror.duckdb") + run, err := newRebuildRun(ctx, local, "test-machine", storage.MirrorPushOptions{}, "other push") + require.NoError(t, err) + lock, _, _, err := claimRebuild(ctx, path, run, false) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, unlockRebuild(lock)) }) + + _, err = Push(ctx, path, local, "test-machine", storage.MirrorPushOptions{}, true, nil) + require.Error(t, err) + _, notStarted := errors.AsType[storage.PushNotStartedError](err) + assert.True(t, notStarted, "the blocked rebuild recorded nothing to resume: %v", err) +} diff --git a/internal/duckdb/sync.go b/internal/duckdb/sync.go index aa1b1d46f8..d837e5f601 100644 --- a/internal/duckdb/sync.go +++ b/internal/duckdb/sync.go @@ -147,6 +147,15 @@ func Push( ctx context.Context, path string, local *db.DB, machine string, opts storage.MirrorPushOptions, full bool, onProgress func(storage.MirrorPushProgress), ) (result storage.MirrorPushResult, err error) { + // A --full push that fails before its rebuild starts left nothing a later + // push resumes. buildMirror reports its own failures before it records + // the rebuild run. + building := false + defer func() { + if err != nil && full && !building { + err = storage.PushNotStartedError{Err: err} + } + }() if err := sweepStaleTempFiles(path); err != nil { log.Printf("duckdbsync: sweeping stale rebuild temp files: %v", err) } @@ -269,6 +278,7 @@ func Push( return storage.MirrorPushResult{}, err } log.Printf("duckdbsync: rebuilding mirror: %s", reason) + building = true result, err = buildMirror(ctx, path, local, machine, opts, run, onProgress) result.Diagnostics.RebuildReason = reason if err == nil { @@ -449,7 +459,7 @@ func (s *Sync) runIncrementalPush( if result.Errors == 0 { if err := s.finalizeIncrementalPush( - ctx, opts, result.Diagnostics.Cutoff, through, + ctx, opts, probe.SourceDatabaseID, result.Diagnostics.Cutoff, through, identityRevision, mappingRevision, ); err != nil { return result, err @@ -479,7 +489,10 @@ func (s *Sync) pushChangedSessions( ctx context.Context, probe MirrorProbe, onProgress func(storage.MirrorPushProgress), result *storage.MirrorPushResult, ) ([]db.Session, []db.Session, error) { - cutoff := time.Now().UTC().Format(localSyncTimestampLayout) + cutoff, err := s.local.MirrorPushCutoff(ctx) + if err != nil { + return nil, nil, err + } result.Diagnostics.Cutoff = cutoff candidates, err := s.local.ListSessionsForMirrorWindow( ctx, probe.LastPushCutoff, nil, nil, @@ -710,19 +723,16 @@ func (s *Sync) readMirrorResidentBatch( // push. Callers must only invoke this after confirming result.Errors == 0: // advancing the cutoff/revisions past a partially failed push would let the // failed sessions silently fall out of the next incremental window. +// +// sourceDatabaseID is the archive generation the push started from: the +// probe's recorded id, which matched the local archive when the push began +// (see rebuildReason). The daemon can replace the archive while a push runs; +// recording the starting id makes the next push rebuild the mirror then. func (s *Sync) finalizeIncrementalPush( - ctx context.Context, opts storage.MirrorPushOptions, cutoff string, + ctx context.Context, opts storage.MirrorPushOptions, + sourceDatabaseID, cutoff string, deletionRevision, identityRevision, mappingRevision int64, ) error { - // The source database id is re-read rather than copied from the probe: - // an incremental push only runs when the probe's recorded id already - // matches the local archive (see rebuildReason), so the two are - // interchangeable, and reading local state keeps this symmetric with - // writeRebuildMetadata. - sourceDatabaseID, err := s.local.GetDatabaseID(ctx) - if err != nil { - return fmt.Errorf("reading local archive database id: %w", err) - } return writeMirrorMetadata(ctx, s.duck, mirrorMetadata{ SchemaVersion: SchemaVersion, DataVersion: db.CurrentDataVersion(), diff --git a/internal/duckdb/sync_fastpath_test.go b/internal/duckdb/sync_fastpath_test.go index af30838290..ff121b162c 100644 --- a/internal/duckdb/sync_fastpath_test.go +++ b/internal/duckdb/sync_fastpath_test.go @@ -235,3 +235,29 @@ func TestDuckCheckpointDecisionRequiresFreeBlockThreshold(t *testing.T) { }) } } + +// The daemon can replace the archive while an incremental push runs. The +// push must record the archive it started from, so the next push sees the +// replacement and rebuilds the mirror. +func TestIncrementalPushRecordsTheArchiveItStartedFrom(t *testing.T) { + ctx := t.Context() + local, path := newPushFixture(t, 1) + _, err := Push(ctx, path, local, "test-machine", storage.MirrorPushOptions{}, false, nil) + require.NoError(t, err) + probe, err := ProbeMirror(ctx, path) + require.NoError(t, err) + started := probe.SourceDatabaseID + + require.NoError(t, local.SetDatabaseIDForTest(ctx, "replaced-during-push")) + syncer := newTestSync(t, path, local, storage.MirrorPushOptions{}) + _, err = syncer.runIncrementalPush(ctx, storage.MirrorPushOptions{}, probe, nil) + require.NoError(t, err) + require.NoError(t, syncer.Close()) + + after, err := ProbeMirror(ctx, path) + require.NoError(t, err) + assert.Equal(t, started, after.SourceDatabaseID) + next, err := Push(ctx, path, local, "test-machine", storage.MirrorPushOptions{}, false, nil) + require.NoError(t, err) + assert.Contains(t, next.Diagnostics.RebuildReason, "source database id changed") +} diff --git a/internal/postgres/handoff_sync_state_test.go b/internal/postgres/handoff_sync_state_test.go new file mode 100644 index 0000000000..8828ac915c --- /dev/null +++ b/internal/postgres/handoff_sync_state_test.go @@ -0,0 +1,76 @@ +package postgres + +import ( + "path/filepath" + "testing" + "testing/synctest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/db" +) + +// A daemon push runs outside the sync lock, so a worker handoff can close the +// archive writer while the push saves its progress. The save must wait for +// the writer instead of failing the push. +func TestHandoffSyncStateWaitsForTheWriter(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + local, err := db.Open(t.Context(), filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, local.Close()) }) + require.NoError(t, local.CloseWriter()) + + done := make(chan error, 1) + go func() { + done <- handoffSyncState{local}.SetSyncState(t.Context(), "last_push_at", "cutoff") + }() + synctest.Wait() + select { + case err := <-done: + require.Failf(t, "saved while the writer was closed", "err: %v", err) + default: + } + require.NoError(t, local.ReopenWriter()) + require.NoError(t, <-done) + + got, err := local.GetSyncState(t.Context(), "last_push_at") + require.NoError(t, err) + assert.Equal(t, "cutoff", got) + }) +} + +// The first push from an archive creates its push marker ID. A worker handoff +// at that moment must not fail the push. +func TestPushMarkerIDWaitsForTheWriter(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + local, err := db.Open(t.Context(), filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, local.Close()) }) + require.NoError(t, local.CloseWriter()) + + type created struct { + id string + err error + } + done := make(chan created, 1) + go func() { + id, err := (&Sync{local: local}).pushMarkerID(t.Context()) + done <- created{id, err} + }() + synctest.Wait() + select { + case got := <-done: + require.Failf(t, "created the ID while the writer was closed", "err: %v", got.err) + default: + } + require.NoError(t, local.ReopenWriter()) + got := <-done + require.NoError(t, got.err) + + stored, err := local.GetSyncState(t.Context(), pushMarkerIDStateKey) + require.NoError(t, err) + assert.Equal(t, stored, got.id) + assert.NotEmpty(t, got.id) + }) +} diff --git a/internal/postgres/push.go b/internal/postgres/push.go index e9b7dca418..11fdb03728 100644 --- a/internal/postgres/push.go +++ b/internal/postgres/push.go @@ -35,6 +35,11 @@ const ( sessionProvenanceBackfillStateKey = "pg_session_provenance_backfill_v2" timestampNormalizationBackfillStateKey = "pg_timestamp_normalization_backfill_v1" unfilteredPublicationScope = "all-projects" + // pushStateSavedKey marks a sync-state scope that has saved its push + // watermark. Archive rebuilds keep it and drop the watermark, so a scope + // with this key and no watermark has lost its push state. A scope that + // never saved one, like a new project filter, lacks the key. + pushStateSavedKey = "pg_push_state_saved_v1" ) // pushMarkerIDStateKey names the local sync-state entry holding this DB's @@ -84,19 +89,35 @@ func (s *Sync) Push( return s.PushWithOptions(ctx, storage.PushOptions{Full: full}, onProgress) } +// defaultFullRunSaveTimeout bounds the progress save after a failed full push. +const defaultFullRunSaveTimeout = 30 * time.Second + // PushWithOptions is Push with per-push options; see storage.PushOptions. func (s *Sync) PushWithOptions( ctx context.Context, opts storage.PushOptions, onProgress func(storage.PushProgress), ) (storage.PushResult, error) { result, err := s.pushWithOptions(ctx, opts, onProgress) - if run := s.fullRun; run != nil { - s.fullRun = nil - if err != nil { - // Keep what the run committed since its last checkpoint. The - // push may have failed because ctx ended, so save without it. - err = errors.Join(err, run.save(context.WithoutCancel(ctx))) - } + run := s.fullRun + s.fullRun = nil + switch { + case err == nil: + case run != nil: + // Keep what the run committed since its last checkpoint. The push + // may have failed because ctx ended, so save without it, but within + // a limit: the save waits for the archive writer, which a stopping + // daemon may never reopen. + timeout := defaultFullRunSaveTimeout + if s.fullRunSaveTimeout > 0 { + timeout = s.fullRunSaveTimeout + } + saveCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeout) + err = errors.Join(err, run.save(saveCtx)) + cancel() + case opts.Full: + // The push failed before it recorded the requested full push, so + // nothing can resume it. + err = storage.PushNotStartedError{Err: err} } return result, err } @@ -230,6 +251,17 @@ func (s *Sync) pushWithOptions( legacyMarkerMachines := pushMarkerLegacyMachines( markerMachine, markerMachineAliases, ) + pushStateSaved, err := state.GetSyncState(ctx, pushStateSavedKey) + if err != nil { + return result, fmt.Errorf("reading %s: %w", pushStateSavedKey, err) + } + // A rebuilt archive keeps this scope's saved-state mark and the archive's + // push marker, but not the push watermark, so PostgreSQL can hold rows + // the rebuild dropped. Only a full push removes them. + if lastPush == "" && boundaryState == "" && pushStateSaved != "" && markerExists { + full = true + noteFull("the local archive lost its record of earlier pushes, as happens after a rebuild") + } var reconciledScopeMoveIDs []string var identityRefreshSessionIDs []string // Keep the backfill marker scoped to target only; all other push @@ -418,6 +450,13 @@ func (s *Sync) pushWithOptions( } } s.fullRun = run + // Take the cutoff before any listing. A sync that commits after an + // earlier listing but stamps before the cutoff would otherwise be missed + // by this push and skipped by every later one. + cutoff, err := s.local.MirrorPushCutoff(ctx) + if err != nil { + return result, err + } if s.isFiltered() { scopeMoveCandidates, scopeErr := listPGProjectScopeMoveCandidates( ctx, s.local, lastPush, @@ -453,8 +492,6 @@ func (s *Sync) pushWithOptions( func() error { return s.syncCursorUsageEvents(ctx) }); err != nil { return result, err } - cutoff := time.Now().UTC().Format(LocalSyncTimestampLayout) - // Candidate selection shares ListSessionsForMirrorWindow with the // DuckDB mirror push: sync_marker >= lastPush, inclusive below and // deliberately unbounded above. An upper bound at cutoff would let a @@ -591,31 +628,16 @@ func (s *Sync) pushWithOptions( } if len(sessions) == 0 { - if s.isFiltered() { - // Filtered pushes use filter-scoped sync state, so - // they can advance their own watermark without - // moving the unfiltered/global cursor. - if err := finalizeFilteredPushState(ctx, - state, lastPush, cutoff, runCommitted, - priorFingerprints, sessionFingerprints, - result.Errors, - ); err != nil { - return result, err - } - } else { - if err := finalizeUnfilteredPushState(ctx, - state, lastPush, cutoff, runCommitted, - priorFingerprints, sessionFingerprints, - result.Errors, - ); err != nil { - return result, err - } - } - if err := persistPushTargetFingerprint(ctx, - state, s.targetFingerprint, + if err := s.savePushState(ctx, state, lastPush, cutoff, runCommitted, + priorFingerprints, sessionFingerprints, result.Errors, ); err != nil { return result, err } + if full && result.Errors == 0 { + if err := s.reconcilePGMissingSessions(ctx, markerID); err != nil { + return result, err + } + } if err := s.writePushMarker( ctx, markerID, markerMachine, markerMachineAliases, ); err != nil { @@ -737,31 +759,17 @@ func (s *Sync) pushWithOptions( } } - if s.isFiltered() { - // Filtered pushes use filter-scoped sync state, so - // they can advance their own watermark without moving - // the unfiltered/global cursor. - if err := finalizeFilteredPushState(ctx, - state, lastPush, cutoff, append(pushed, runCommitted...), - priorFingerprints, sessionFingerprints, - result.Errors, - ); err != nil { - return result, err - } - } else { - if err := finalizeUnfilteredPushState(ctx, - state, lastPush, cutoff, append(pushed, runCommitted...), - priorFingerprints, sessionFingerprints, - result.Errors, - ); err != nil { - return result, err - } - } - if err := persistPushTargetFingerprint(ctx, - state, s.targetFingerprint, + if err := s.savePushState(ctx, state, lastPush, cutoff, + append(pushed, runCommitted...), priorFingerprints, sessionFingerprints, + result.Errors, ); err != nil { return result, err } + if full && result.Errors == 0 { + if err := s.reconcilePGMissingSessions(ctx, markerID); err != nil { + return result, err + } + } // Write the push marker only after the push and local finalization // succeed. A reset-recovery push that fails before this point leaves // the marker absent, so the next push re-detects the reset and retries @@ -1252,10 +1260,10 @@ func normalizePushMarkerMachineAliases( // name, so a machine rename keeps the same marker, and unique per local DB, so // a different host pushing to the same PG cannot mask this host's reset. func (s *Sync) pushMarkerID(ctx context.Context) (string, error) { - state := s.local - if state == nil { + if s.local == nil { return "", errors.New("local db is required") } + state := handoffSyncState{s.local} id, err := state.GetSyncState(ctx, pushMarkerIDStateKey) if err != nil { return "", fmt.Errorf("reading push marker id: %w", err) @@ -1824,6 +1832,50 @@ func persistPushSourceArchiveID(ctx context.Context, local syncStateStore, archi return nil } +// savePushState saves the push watermark, boundary state, and target +// fingerprint in one archive transaction, and only while the archive is the +// one the push read. A /resync or compaction can replace the archive during +// a push; the push then mixed rows from two archives, and its cutoff would +// hide rows the rebuild wrote. The replacement starts without push state, so +// the next push runs full. +func (s *Sync) savePushState(ctx context.Context, + state syncStateStore, + lastPush, cutoff string, + sessions []db.Session, + priorFingerprints map[string]string, + sessionFingerprints map[string]string, + sessionErrors int, +) error { + batch := newSyncStateBatch() + // Filtered pushes use filter-scoped sync state, so they can advance + // their own watermark without moving the unfiltered/global cursor. + finalize := finalizeUnfilteredPushState + if s.isFiltered() { + finalize = finalizeFilteredPushState + } + if err := finalize(ctx, batch, lastPush, cutoff, sessions, + priorFingerprints, sessionFingerprints, sessionErrors, + ); err != nil { + return err + } + if err := persistPushTargetFingerprint(ctx, batch, s.targetFingerprint); err != nil { + return err + } + if err := batch.SetSyncState(ctx, pushStateSavedKey, "1"); err != nil { + return err + } + writer, ok := state.(databaseSyncStateWriter) + if !ok { + return fmt.Errorf("sync state store %T cannot save push state in one transaction", state) + } + err := writer.SetSyncStatesForDatabase(ctx, s.databaseGeneration, batch.values) + if errors.Is(err, db.ErrDatabaseReplaced) { + return errors.New("the local archive was rebuilt while this push ran. " + + "The next push will resend every session") + } + return err +} + func pushTargetState( lastPush, boundaryState, storedTargetFingerprint, currentTargetFingerprint string, @@ -2043,6 +2095,91 @@ func purgePGExcludedPushSessions( return deletePGExcludedSessionRows(ctx, pg, purgeIDs) } +// Reconcile after publishing sessions so Codex page curation transfers finish +// before deletion cascades. The unfinished full run keeps failed cleanup retryable. +func (s *Sync) reconcilePGMissingSessions(ctx context.Context, ownerMarker string) error { + // Empty Codex thread anchors still own remote restore/purge scope. + where := "owner_marker = $1 AND NOT (trash_includes_codex_pages AND deleted_at IS NOT NULL)" + args := []any{ownerMarker} + if len(s.projects) > 0 { + args = append(args, s.projects) + where += fmt.Sprintf(" AND project = ANY($%d)", len(args)) + } + if len(s.excludeProjects) > 0 { + args = append(args, s.excludeProjects) + where += fmt.Sprintf(" AND project <> ALL($%d)", len(args)) + } + rows, err := s.pg.QueryContext(ctx, "SELECT id FROM sessions WHERE "+where, args...) + if err != nil { + return fmt.Errorf("listing owned pg sessions for full push reconciliation: %w", err) + } + defer rows.Close() + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return fmt.Errorf("scanning owned pg session for full push reconciliation: %w", err) + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + return err + } + if err := rows.Close(); err != nil { + return err + } + return db.QueryChunkedSize(ids, pushComparisonBatchSize, func(chunk []string) error { + // Read live presence, including trashed and out-of-scope local rows. + // A sync may have added a session since the push listed its candidates. + present, err := s.local.SessionProjectsByIDs(ctx, chunk) + if err != nil { + return err + } + var missing []string + for _, id := range chunk { + if _, exists := present[id]; !exists { + missing = append(missing, id) + } + } + if len(missing) == 0 { + return nil + } + tx, err := s.pg.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + // Recheck ownership and scope, and clear vectors only for deleted rows. + rows, err := tx.QueryContext(ctx, + "DELETE FROM sessions WHERE "+where+fmt.Sprintf(" AND id = ANY($%d) RETURNING id", len(args)+1), + append(slices.Clone(args), missing)...) + if err != nil { + return fmt.Errorf("deleting pg sessions absent from the archive: %w", err) + } + defer rows.Close() + var deleted []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return err + } + deleted = append(deleted, id) + } + if err := rows.Err(); err != nil { + return err + } + if err := rows.Close(); err != nil { + return err + } + for _, id := range deleted { + if err := clearSessionVectorsTx(ctx, tx, id); err != nil { + return err + } + } + return tx.Commit() + }) +} + func (s *Sync) reconcilePGProjectScopeMoves( ctx context.Context, ownerMarker string, diff --git a/internal/postgres/push_generation_pgtest_test.go b/internal/postgres/push_generation_pgtest_test.go new file mode 100644 index 0000000000..b021846be5 --- /dev/null +++ b/internal/postgres/push_generation_pgtest_test.go @@ -0,0 +1,202 @@ +//go:build pgtest + +package postgres + +import ( + "context" + "errors" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/storage" +) + +// rebuildArchive stands in for a /resync: the replacement archive gets a new +// database ID and starts without this archive's PostgreSQL push state. +func (f resumeFixture) rebuildArchive(t *testing.T, id string) { + t.Helper() + require.NoError(t, f.local.SetDatabaseIDForTest(t.Context(), id)) + require.NoError(t, clearPushState(t.Context(), f.local)) +} + +const lostStateReason = "the local archive lost its record of earlier pushes, as happens after a rebuild" + +// A rebuild keeps this archive's push marker but drops its watermark, so +// PostgreSQL can hold rows the rebuild dropped. The next push must be full. +func TestPushAfterArchiveRebuildIsFull(t *testing.T) { + f := newResumeFixture(t, "agentsview_push_generation_test") + _, err := f.sync().Push(t.Context(), false, nil) + require.NoError(t, err) + res, err := f.sync().Push(t.Context(), false, nil) + require.NoError(t, err) + require.False(t, res.Full) + + f.rebuildArchive(t, "rebuilt") + res, err = f.sync().Push(t.Context(), false, nil) + require.NoError(t, err) + assert.True(t, res.Full) + assert.Equal(t, lostStateReason, res.FullReason) +} + +func TestPushAfterArchiveRebuildRemovesMissingSessions(t *testing.T) { + for _, tc := range []struct { + name string + projects []string + excludeProjects []string + empty bool + interrupt bool + want []string + }{ + {name: "unfiltered", want: []string{"foreign", "legacy", "new"}}, + {name: "empty archive", empty: true, want: []string{"foreign", "legacy"}}, + {name: "resumed push", interrupt: true, want: []string{"foreign", "legacy", "new"}}, + {name: "included projects", projects: []string{"proj"}, want: []string{"foreign", "legacy", "new", "outside"}}, + {name: "excluded projects", excludeProjects: []string{"outside"}, want: []string{"foreign", "legacy", "new", "outside"}}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newResumeFixture(t, "agentsview_push_generation_test") + for _, id := range []string{"foreign", "legacy", "outside"} { + project := "proj" + if id == "outside" { + project = "outside" + } + require.NoError(t, f.local.UpsertSession(t.Context(), db.Session{ + ID: id, Project: project, Machine: "machine", Agent: "claude", + })) + } + res, err := f.sync().Push(t.Context(), false, nil) + require.NoError(t, err) + require.Zero(t, res.Errors) + // Establish this filter's push history before the rebuild drops it. + s := f.sync() + s.projects, s.excludeProjects = tc.projects, tc.excludeProjects + s.syncState = newScopedSyncStateStore(handoffSyncState{f.local}, + pushSyncStateScope("", tc.projects, tc.excludeProjects), false) + _, err = s.Push(t.Context(), false, nil) + require.NoError(t, err) + _, err = f.pg.ExecContext(t.Context(), `UPDATE sessions SET owner_marker = 'another-archive' WHERE id = 'foreign'`) + require.NoError(t, err) + _, err = f.pg.ExecContext(t.Context(), `UPDATE sessions SET owner_marker = '' WHERE id = 'legacy'`) + require.NoError(t, err) + + rebuilt, err := db.Open(t.Context(), filepath.Join(t.TempDir(), "rebuilt.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, rebuilt.Close()) }) + require.NoError(t, rebuilt.CopyArchiveIdentityFrom(f.local.Path())) + require.NoError(t, rebuilt.CopySyncStateFrom(f.local.Path())) + require.NoError(t, f.local.CloseConnections(t.Context())) + omitted := []string{"old", "mid", "foreign", "legacy", "outside"} + if tc.empty { + omitted = append(omitted, "new") + } + // Parser exclusions during resync omit rows without adding user deletions. + _, err = rebuilt.CopyOrphanedDataFromExcluding(f.local.Path(), omitted) + require.NoError(t, err) + f.local = rebuilt + s = f.sync() + s.projects, s.excludeProjects = tc.projects, tc.excludeProjects + s.syncState = newScopedSyncStateStore(handoffSyncState{f.local}, + pushSyncStateScope("", tc.projects, tc.excludeProjects), false) + if tc.interrupt { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + s.afterPushBatch = cancel + _, err = s.Push(ctx, false, nil) + require.ErrorIs(t, err, context.Canceled) + assert.Contains(t, f.sessionIDs(t), "old", "interrupted publication must leave cleanup for the retry") + s.afterPushBatch = nil + } + res, err = s.Push(t.Context(), false, nil) + require.NoError(t, err) + assert.True(t, res.Full) + assert.Zero(t, res.Errors) + assert.Equal(t, tc.want, f.sessionIDs(t)) + var messages int + require.NoError(t, f.pg.QueryRowContext(t.Context(), + `SELECT count(*) FROM messages WHERE session_id IN ('old', 'mid')`).Scan(&messages)) + assert.Zero(t, messages, "removed sessions must not leave transcripts behind") + }) + } +} + +// Push state is kept per project filter, and the push marker covers the whole +// archive. The first push with a new filter has saved no state of its own, so +// it must not claim it lost any. +func TestFirstFilteredPushDoesNotClaimLostState(t *testing.T) { + f := newResumeFixture(t, "agentsview_push_generation_test") + _, err := f.sync().Push(t.Context(), false, nil) + require.NoError(t, err) + + filtered := f.sync() + filtered.projects = []string{"proj"} + filtered.syncState = newScopedSyncStateStore(handoffSyncState{f.local}, + pushSyncStateScope("", filtered.projects, nil), false) + res, err := filtered.Push(t.Context(), false, nil) + require.NoError(t, err) + assert.NotEqual(t, lostStateReason, res.FullReason) +} + +// A rebuild during a push mixes rows from two archives. The push must fail +// without saving its cutoff into the replacement, and the next push must be +// full. +func TestPushFailsWhenTheArchiveIsRebuiltDuringIt(t *testing.T) { + f := newResumeFixture(t, "agentsview_push_generation_test") + _, err := f.sync().Push(t.Context(), false, nil) + require.NoError(t, err) + require.NoError(t, f.local.InsertMessages(t.Context(), []db.Message{{ + SessionID: "new", Ordinal: 1, Role: "user", Content: "again", + ContentLength: 5, Timestamp: "2026-03-02T00:00:00Z", + }})) + + s := f.sync() + s.afterPushBatch = func() { f.rebuildArchive(t, "rebuilt") } + _, err = s.Push(t.Context(), false, nil) + require.ErrorContains(t, err, "rebuilt while this push ran") + saved, err := f.local.GetSyncState(t.Context(), "last_push_at") + require.NoError(t, err) + assert.Empty(t, saved, "the push must not save its cutoff into the replacement") + + res, err := f.sync().Push(t.Context(), false, nil) + require.NoError(t, err) + assert.True(t, res.Full) + assert.Equal(t, lostStateReason, res.FullReason) +} + +// A --full push that fails before it records its full push reports that it +// never started, so a watch keeps the --full request. +func TestFullPushFailingBeforeItsRunIsNotStarted(t *testing.T) { + f := newResumeFixture(t, "agentsview_push_generation_test") + ctx, cancel := context.WithCancel(t.Context()) + cancel() + for _, full := range []bool{false, true} { + _, err := f.sync().PushWithOptions(ctx, storage.PushOptions{Full: full}, nil) + require.ErrorIs(t, err, context.Canceled) + _, notStarted := errors.AsType[storage.PushNotStartedError](err) + assert.Equal(t, full, notStarted, "full=%t", full) + } +} + +// A failed full push saves its progress without the push's cancellation, so +// the save must still end when the archive writer never reopens, as during a +// daemon shutdown. +func TestFailedFullPushProgressSaveEnds(t *testing.T) { + f := newResumeFixture(t, "agentsview_push_generation_test") + s := f.sync() + s.fullRunSaveTimeout = 100 * time.Millisecond + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + s.afterPushBatch = func() { + require.NoError(t, f.local.CloseWriter()) + cancel() + } + _, err := s.PushWithOptions(ctx, storage.PushOptions{Full: true}, nil) + require.ErrorIs(t, err, context.Canceled) + require.ErrorIs(t, err, context.DeadlineExceeded, + "the progress save gives up instead of waiting forever") + require.NoError(t, f.local.ReopenWriter()) +} diff --git a/internal/postgres/push_pgtest_test.go b/internal/postgres/push_pgtest_test.go index cc6a045c89..9c43958031 100644 --- a/internal/postgres/push_pgtest_test.go +++ b/internal/postgres/push_pgtest_test.go @@ -7,6 +7,7 @@ import ( "database/sql" "fmt" "path/filepath" + "runtime" "testing" "time" @@ -17,6 +18,7 @@ import ( "go.kenn.io/agentsview/internal/export" "go.kenn.io/agentsview/internal/money" "go.kenn.io/agentsview/internal/storage" + "go.kenn.io/agentsview/internal/testutil" ) func TestPushPreservesLegacyOffsetTimestamps(t *testing.T) { @@ -1120,10 +1122,7 @@ func TestPushMessageNanosecondTimestampNoRewriteRegression(t *testing.T) { "2026-01-01T00:00:00.123456Z") ctidBefore := pgMessageCTID(t, pg, sessID, 1) - require.NoError(t, localDB.SetSyncState(t.Context(), "last_push_at", ""), - "clearing last_push_at") - require.NoError(t, localDB.SetSyncState(t.Context(), lastPushBoundaryStateKey, ""), - "clearing boundary state") + forgetPushHistory(t, localDB) _, err = sync.Push(ctx, false, nil) require.NoError(t, err, "Push same timestamp") @@ -2353,6 +2352,20 @@ func assertPGMessageTimestamp( assert.Equal(t, want, FormatISO8601(gotTime.Time)) } +// forgetPushHistory makes the next push look like this archive's first push to +// a database that already holds its sessions, so that push compares +// fingerprints. Clearing only the watermark would force a full push, because +// the saved-state mark shows this archive pushed before. +func forgetPushHistory(t *testing.T, localDB *db.DB) { + t.Helper() + require.NoError(t, localDB.SetSyncState(t.Context(), "last_push_at", ""), + "clearing last_push_at") + require.NoError(t, localDB.SetSyncState(t.Context(), lastPushBoundaryStateKey, ""), + "clearing boundary state") + require.NoError(t, localDB.SetSyncState(t.Context(), pushStateSavedKey, ""), + "clearing the saved-state mark") +} + func pgMessageCTID( t *testing.T, pg *sql.DB, @@ -2459,11 +2472,7 @@ func TestPushMessagesSanitizesNULBytes(t *testing.T) { WHERE session_id = $1 AND ordinal = 0`, sessID, ).Scan(&ctidBefore), "reading ctid before second push") - require.NoError(t, localDB.SetSyncState(t.Context(), "last_push_at", ""), - "clearing last_push_at") - require.NoError(t, - localDB.SetSyncState(t.Context(), lastPushBoundaryStateKey, ""), - "clearing boundary state") + forgetPushHistory(t, localDB) res, err = sync.Push(ctx, false, nil) require.NoError(t, err, "Push (second)") @@ -3701,3 +3710,88 @@ func TestPGPushReplacesChangedUsage(t *testing.T) { assert.Equal(t, 1, result.Errors, "a malformed usage timestamp must fail the push and remain retryable") } + +// A filtered push that starts while a local sync is moving a session out of +// scope must reconcile the move. The push lists move candidates and takes its +// cutoff; if it listed before the move committed and took the cutoff after, +// the session would stay remote under its old project with the watermark +// already past the move. +func TestFilteredPushReconcilesProjectMoveCommittedDuringPush(t *testing.T) { + const ( + sessionID = "pg-concurrent-move" + sourceProject = "source_project" + targetProject = "target_project" + schema = "agentsview_concurrent_move" + ) + pgURL := testPGURL(t) + cleanNamedPGSchema(t, pgURL, schema) + t.Cleanup(func() { cleanNamedPGSchema(t, pgURL, schema) }) + ctx := t.Context() + local, err := db.Open(ctx, filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, local.Close()) }) + startedAt := "2026-07-16T12:00:00.000Z" + require.NoError(t, local.UpsertSession(ctx, db.Session{ + ID: sessionID, Project: sourceProject, Machine: "local", + Agent: "claude", StartedAt: &startedAt, LocalModifiedAt: &startedAt, + MessageCount: 1, UserMessageCount: 1, + })) + syncer, err := New(pgURL, schema, local, "test-machine", true, storage.PusherOptions{ + Projects: []string{sourceProject}, + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, syncer.Close()) }) + _, err = syncer.Push(ctx, true, nil) + require.NoError(t, err) + + // A local sync moves the session to an excluded project and holds its + // transaction open. Its stamp is after the first push's cutoff. + stamp := time.Now().UTC().Format("2006-01-02T15:04:05.000Z") + moved := make(chan struct{}) + release := make(chan struct{}) + committed := make(chan error, 1) + go func() { + committed <- local.Update(ctx, func(tx *sql.Tx) error { + if _, err := tx.ExecContext(ctx, ` + UPDATE sessions SET project = ?, local_modified_at = ? + WHERE id = ?`, targetProject, stamp, sessionID, + ); err != nil { + return err + } + close(moved) + <-release + return nil + }) + }() + <-moved + + pushed := make(chan error, 1) + go func() { + _, err := syncer.Push(ctx, false, nil) + pushed <- err + }() + // Commit the move once the push waits for its cutoff, or once it has + // finished without waiting. + var pushErr error + finished := false + for !finished && !testutil.GoroutineBlockedOnMutexIn("(*DB).MirrorPushCutoff") { + select { + case pushErr = <-pushed: + finished = true + default: + runtime.Gosched() + } + } + close(release) + require.NoError(t, <-committed) + if !finished { + pushErr = <-pushed + } + require.NoError(t, pushErr) + + var count int + require.NoError(t, syncer.pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE id = $1`, sessionID, + ).Scan(&count)) + assert.Zero(t, count, "a session moved out of scope during the push must leave the remote") +} diff --git a/internal/postgres/sync.go b/internal/postgres/sync.go index 6683570acd..041471b6be 100644 --- a/internal/postgres/sync.go +++ b/internal/postgres/sync.go @@ -21,6 +21,84 @@ import ( type syncStateStore = storage.SyncStateStore +// handoffSyncState writes push state to the local archive and rides out a +// worker handoff. The daemon closes its archive writer while a sync worker +// runs, and a daemon push runs outside the sync lock, so a handoff can start +// during a push. A write that hits a closed writer waits for it to reopen and +// tries again. +type handoffSyncState struct{ local *db.DB } + +func (h handoffSyncState) GetSyncState(ctx context.Context, key string) (string, error) { + return h.local.GetSyncState(ctx, key) +} + +func (h handoffSyncState) SetSyncState(ctx context.Context, key, value string) error { + _, err := retryAcrossHandoff(ctx, h.local, func() (struct{}, error) { + return struct{}{}, h.local.SetSyncState(ctx, key, value) + }) + return err +} + +func (h handoffSyncState) GetOrCreateSyncState( + ctx context.Context, key, defaultValue string, +) (string, error) { + return retryAcrossHandoff(ctx, h.local, func() (string, error) { + return h.local.GetOrCreateSyncState(ctx, key, defaultValue) + }) +} + +// databaseSyncStateWriter writes several sync-state values in one archive +// transaction, and only while the archive's database ID is still databaseID. +type databaseSyncStateWriter interface { + SetSyncStatesForDatabase(ctx context.Context, databaseID string, values map[string]string) error +} + +func (h handoffSyncState) SetSyncStatesForDatabase( + ctx context.Context, databaseID string, values map[string]string, +) error { + _, err := retryAcrossHandoff(ctx, h.local, func() (struct{}, error) { + return struct{}{}, h.local.SetSyncStatesForDatabase(ctx, databaseID, values) + }) + return err +} + +// syncStateBatch collects sync-state writes so a caller can commit them +// together. It is write-only. +type syncStateBatch struct{ values map[string]string } + +func newSyncStateBatch() *syncStateBatch { + return &syncStateBatch{values: make(map[string]string)} +} + +var errSyncStateBatchWriteOnly = errors.New("sync state batch is write-only") + +func (b *syncStateBatch) GetSyncState(context.Context, string) (string, error) { + return "", errSyncStateBatchWriteOnly +} + +func (b *syncStateBatch) SetSyncState(_ context.Context, key, value string) error { + b.values[key] = value + return nil +} + +func (b *syncStateBatch) GetOrCreateSyncState(context.Context, string, string) (string, error) { + return "", errSyncStateBatchWriteOnly +} + +func retryAcrossHandoff[T any]( + ctx context.Context, local *db.DB, write func() (T, error), +) (T, error) { + for { + value, err := write() + if !errors.Is(err, db.ErrWriterClosed) { + return value, err + } + if err := local.WaitForWriter(ctx); err != nil { + return value, err + } + } +} + type scopedSyncStateStore struct { base syncStateStore scope string @@ -117,6 +195,23 @@ func (s *scopedSyncStateStore) SetSyncState(ctx context.Context, return s.base.SetSyncState(ctx, s.scopedKey(key), value) } +func (s *scopedSyncStateStore) SetSyncStatesForDatabase( + ctx context.Context, databaseID string, values map[string]string, +) error { + if err := s.ensureMigration(ctx); err != nil { + return err + } + writer, ok := s.base.(databaseSyncStateWriter) + if !ok { + return fmt.Errorf("sync state store %T cannot write in one transaction", s.base) + } + scoped := make(map[string]string, len(values)) + for key, value := range values { + scoped[s.scopedKey(key)] = value + } + return writer.SetSyncStatesForDatabase(ctx, databaseID, scoped) +} + func (s *scopedSyncStateStore) GetOrCreateSyncState(ctx context.Context, key, defaultValue string, ) (string, error) { @@ -200,6 +295,9 @@ type Sync struct { // fullRunCheckpointInterval, when positive, replaces // fullPushRunCheckpointInterval. fullRunCheckpointInterval time.Duration + // fullRunSaveTimeout, when positive, replaces + // defaultFullRunSaveTimeout. + fullRunSaveTimeout time.Duration // pushBatchSize, when positive, replaces the session batch size. pushBatchSize int // afterPushBatch is a test hook called after each session batch. @@ -216,7 +314,7 @@ func (s *Sync) effectiveSyncState() syncStateStore { if s.syncState != nil { return s.syncState } - return s.local + return handoffSyncState{s.local} } func (s *Sync) aliasBackfillSyncStateOrDefault() syncStateStore { @@ -287,12 +385,12 @@ func New( pg: pg, local: local, syncState: newScopedSyncStateStore( - local, + handoffSyncState{local}, syncStateScope, migrateLegacySyncState, ), aliasBackfillState: newScopedSyncStateStore( - local, + handoffSyncState{local}, aliasBackfillStateScope, false, ), diff --git a/internal/server/huma_routes_push.go b/internal/server/huma_routes_push.go index 798ba38099..fa1266d9a4 100644 --- a/internal/server/huma_routes_push.go +++ b/internal/server/huma_routes_push.go @@ -8,9 +8,11 @@ import ( "net/http" "path/filepath" "strings" + gosync "sync" "time" "github.com/danielgtaylor/huma/v2" + "go.kenn.io/agentsview/internal/archivepush" "go.kenn.io/agentsview/internal/config" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/parser" @@ -107,43 +109,70 @@ func (s *Server) registerPushRoutes() { // runPushStream executes run once and writes its outcome to the client: SSE // progress/done/error events when the request negotiated an event stream // (the CLI's daemon-delegated push renders these live), a single JSON body -// otherwise. run receives the progress callback to thread into the push; -// it is a throttled SSE sender in stream mode and nil in JSON mode — the -// daemon-side debug.log progress logger is composed by the caller. +// otherwise. run receives two senders, both nil in JSON mode: progress, a +// throttled sender to thread into the push, and phase, an unthrottled sender +// for the waiting and rebuilt reports a client must not miss. The +// daemon-side debug.log progress logger is composed by the caller. An error +// body carries push_started, so a watch client can keep a --full request +// that no push started. func runPushStream[T any]( hctx huma.Context, - run func(onProgress func(T)) (any, error), + run func(progress, phase func(T)) (any, error), ) { if strings.Contains(hctx.Header("Accept"), "text/event-stream") { if sse, ok := newHumaSSEStream(hctx); ok { send := newPushProgressStreamSender(func(p T) { sse.SendJSON("progress", p) }) - result, err := run(send) + result, err := run(send, func(p T) { + sse.SendJSON("progress", p) + }) if err != nil { - sse.SendJSON("error", map[string]string{"error": err.Error()}) + sse.SendJSON("error", pushErrorBody(err.Error(), err)) return } sse.SendJSON("done", result) return } } - result, err := run(nil) + result, err := run(nil, nil) if err != nil { if errors.Is(err, db.ErrWriterClosed) { server, ok := hctx.Context().Value(ctxKeyHumaServer).(*Server) response := writerClosedResponse(hctx.AppendHeader, ok && server.archiveStale()) - writeHumaJSON(hctx, response.Status, - map[string]string{"error": response.Message}) + writeHumaJSON(hctx, response.Status, pushErrorBody(response.Message, err)) return } writeHumaJSON(hctx, http.StatusInternalServerError, - map[string]string{"error": err.Error()}) + pushErrorBody(err.Error(), err)) return } writeHumaJSON(hctx, http.StatusOK, result) } +// pushErrorResponse is the error body of a push route. +type pushErrorResponse struct { + Error string `json:"error"` + // PushStarted is false when the client should ask for --full again: the + // push failed before it recorded a full push the mirror could resume, and + // the request did not rebuild the archive, which makes every mirror push + // in full on its own. + PushStarted bool `json:"push_started"` +} + +func pushErrorBody(message string, err error) pushErrorResponse { + _, notStarted := errors.AsType[storage.PushNotStartedError](err) + _, rebuilt := errors.AsType[archiveRebuiltError](err) + return pushErrorResponse{Error: message, PushStarted: !notStarted || rebuilt} +} + +// logPushPhase logs a daemon push that has to wait for another push. +func logPushPhase(name, phase string) { + if phase == storage.PushPhaseWaiting { + log.Printf("%s push: waiting for another push to finish", name) + } +} + // composePushProgress fans one progress report out to each non-nil callback. func composePushProgress[P any](fns ...func(P)) func(P) { return func(p P) { @@ -462,18 +491,59 @@ func watchPathWithinRoots(path string, roots []string) bool { return false } +// pushTurn lets one daemon push run at a time. Without it, two pushes to the +// same destination could overlap: the slower one could write an older copy of +// a session after the newer one recorded its cutoff, and later incremental +// pushes would skip that session. The zero value is ready to use. +type pushTurn struct { + once gosync.Once + turn chan struct{} +} + +// take waits for this push's turn and returns the func that ends it. It calls +// onWait, when set, if the push must wait for another push. It returns ctx's +// error if ctx ends first. +func (p *pushTurn) take(ctx context.Context, onWait func()) (func(), error) { + p.once.Do(func() { p.turn = make(chan struct{}, 1) }) + end := func() { <-p.turn } + select { + case p.turn <- struct{}{}: + return end, nil + default: + } + if onWait != nil { + onWait() + } + select { + case p.turn <- struct{}{}: + return end, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + // syncThenRunForPush brings the local archive current and then runs the push -// work serialized against sync passes. Full and stale-archive pushes are -// archive-scale: with the worker-backed resync runner wired they route the -// rebuild through the worker (matching /api/v1/resync and the pre-session-sync -// resync) instead of running an in-process resync in the long-lived daemon, -// and the push work then runs under the engine's exclusive sync lock — after -// the deferred-signal flush — so it still observes the post-rebuild archive -// and stays serialized against watcher and periodic syncs. A validated batch -// on a current archive uses SyncWatchBatchThenRun, applying only its bounded -// changed scope before the push under that same lock. Requests without a batch -// retain SyncThenRun. Without a runner (tests, remote-mode-less setups), full -// and stale cases keep the in-process SyncThenRun path. +// work. The push runs after the sync, not under its lock: watcher and periodic +// syncs keep writing the archive while a long push copies it, and each push +// takes its cutoff under the archive write lock so later pushes pick those +// writes up. Pushes take turns (see pushTurn), and a push takes its turn +// before its own sync, so its archive rebuild never replaces the archive +// while another push reads it. onPhase, when set, is called with +// storage.PushPhaseWaiting if the push must wait for its turn, and with +// storage.PushPhaseRebuilt as soon as its sync has rebuilt the archive. +// Every mirror pushes in full after a rebuild, so a client whose stream drops +// after that report need not ask for --full again. Before the work it +// waits for any running sync and flushes deferred signal recomputes, so the +// push copies settled sessions. +// An error from before work runs is a storage.PushNotStartedError. +// +// Full and stale-archive pushes are archive-scale: with the worker-backed +// resync runner wired they route the rebuild through the worker (matching +// /api/v1/resync and the pre-session-sync resync) instead of running an +// in-process resync in the long-lived daemon. A validated batch on a current +// archive applies only its bounded changed scope. Requests without a batch +// run a normal sync. Without a runner (tests, remote-mode-less setups), full +// and stale cases keep the in-process resync. func (s *Server) syncThenRunForPush( ctx context.Context, engine *syncpkg.Engine, @@ -481,60 +551,136 @@ func (s *Server) syncThenRunForPush( full bool, watchBatch *syncpkg.WatchBatch, watchRecovery *syncpkg.WatchRecoveryScope, + onPhase func(phase string), work func(forceFull bool) error, ) error { + if onPhase == nil { + onPhase = func(string) {} + } + endTurn, err := s.pushTurn.take(ctx, func() { onPhase(storage.PushPhaseWaiting) }) + if err != nil { + return storage.PushNotStartedError{Err: err} + } + defer endTurn() + startDatabaseID, err := local.GetDatabaseID(ctx) + if err != nil { + return storage.PushNotStartedError{Err: err} + } + forceFull, incomplete, err := s.syncForPush( + ctx, engine, local, full, watchBatch, watchRecovery, + ) + rebuilt, idErr := archiveRebuiltSince(ctx, local, startDatabaseID) + if idErr != nil { + return storage.PushNotStartedError{Err: errors.Join(err, idErr)} + } + if rebuilt { + onPhase(storage.PushPhaseRebuilt) + } + if err != nil { + return markArchiveRebuilt(ctx, local, startDatabaseID, storage.PushNotStartedError{Err: err}) + } + if err := engine.SettleForPush(ctx); err != nil { + return markArchiveRebuilt(ctx, local, startDatabaseID, storage.PushNotStartedError{Err: err}) + } + pushErr := markArchiveRebuilt(ctx, local, startDatabaseID, work(forceFull)) + if incomplete == nil { + return pushErr + } + if pushErr != nil { + return errors.Join(incomplete, pushErr) + } + log.Printf("local ingestion warning during archive push: %v", incomplete) + return nil +} + +// archiveRebuiltError marks a push error from a request that rebuilt the +// archive. Every mirror pushes in full after a rebuild, so the client need +// not ask for --full again; asking would rebuild the archive on every retry. +type archiveRebuiltError struct{ err error } + +func (e archiveRebuiltError) Error() string { return e.err.Error() } +func (e archiveRebuiltError) Unwrap() error { return e.err } + +// markArchiveRebuilt marks err when the archive's database ID changed since +// startDatabaseID, which a rebuild does. +func markArchiveRebuilt( + ctx context.Context, local *db.DB, startDatabaseID string, err error, +) error { + if err == nil { + return nil + } + rebuilt, idErr := archiveRebuiltSince(ctx, local, startDatabaseID) + if idErr != nil { + return errors.Join(err, idErr) + } + if rebuilt { + return archiveRebuiltError{err} + } + return err +} + +// archiveRebuiltSince reports whether the archive's database ID changed since +// startDatabaseID, which a rebuild does. It reads the ID after ctx ends too, +// so a canceled push still learns whether it rebuilt the archive. +func archiveRebuiltSince( + ctx context.Context, local *db.DB, startDatabaseID string, +) (bool, error) { + id, err := local.GetDatabaseID(context.WithoutCancel(ctx)) + if err != nil { + return false, fmt.Errorf("reading archive database id: %w", err) + } + return id != startDatabaseID, nil +} + +// syncForPush runs the sync a push needs and reports whether the push must +// resend every session. incomplete is set when some sources failed but the +// archive is still current, so the push may copy it. +func (s *Server) syncForPush( + ctx context.Context, + engine *syncpkg.Engine, + local *db.DB, + full bool, + watchBatch *syncpkg.WatchBatch, + watchRecovery *syncpkg.WatchRecoveryScope, +) (forceFull bool, incomplete, err error) { if watchBatch != nil && !full && !local.NeedsResync() { stats, err := engine.SyncWatchBatchThenRun( - ctx, *watchBatch, watchRecovery, - func() error { return work(false) }, + ctx, *watchBatch, watchRecovery, func() error { return nil }, ) if err == nil { err = requireProcessingComplete(stats) } - return err + return false, nil, err } if s.localResyncRunner == nil || (!full && !local.NeedsResync()) { currentArchive := !full && !local.NeedsResync() - stats, err := engine.SyncThenRun(ctx, full, nil, work) + stats, err := engine.SyncThenRun(ctx, full, nil, + func(f bool) error { + forceFull = f + return nil + }) if err != nil || stats.ProcessingComplete() { - return err + return forceFull, nil, err } + // Local sync owns retries for failed sources. An unscoped push must + // still report its own outcome, so the client does not repeat a + // completed copy or lose row-level errors from its result. incomplete := requireProcessingComplete(stats) - if !currentArchive { - return incomplete + if !currentArchive || local.NeedsResync() { + return false, nil, incomplete } - // Local sync owns retries for failed sources. An unscoped mirror push - // must report its own outcome so the client does not repeat a completed - // copy or lose row-level errors from its result. SyncThenRun skips work - // on incomplete processing, so copy the archive under its lock here. - pushErr := engine.RunExclusiveFlushed(func() error { - if err := ctx.Err(); err != nil { - return err - } - if local.NeedsResync() { - return incomplete - } - return work(false) - }) - if pushErr != nil { - return errors.Join(incomplete, pushErr) - } - log.Printf("local ingestion warning during archive push: %v", incomplete) - return nil + return false, incomplete, nil } if _, err := s.runResyncWithFallback(ctx, engine, nil); err != nil { - return err - } - if err := ctx.Err(); err != nil { - return err + return false, nil, err } - return engine.RunExclusiveFlushed(func() error { return work(true) }) + return true, nil, ctx.Err() } // replicaPushHandler builds the daemon-delegated push handler for one // replica backend. The handler validates the request before the stream body // flushes a 200, brings the archive current, then runs the backend's pusher -// against the daemon's archive under the sync lock. +// against the daemon's archive while local sync keeps running. func (s *Server) replicaPushHandler( replica storage.Replica, ) func(context.Context, *daemonPushInput) (*huma.StreamResponse, error) { @@ -588,52 +734,60 @@ func (s *Server) humaReplicaPush( body := in.Body return &huma.StreamResponse{Body: func(hctx huma.Context) { runPushStream(hctx, func( - streamProgress func(storage.PushProgress), + streamProgress, streamPhase func(storage.PushProgress), ) (any, error) { onProgress := composePushProgress( newReplicaPushProgressLogger(name), streamProgress, ) + reportPhase := func(phase string) { + logPushPhase(name, phase) + if streamPhase != nil { + streamPhase(storage.PushProgress{Phase: phase}) + } + } + pushCtx := db.WithWriterWaitNotice(ctx, func() { + reportPhase(storage.PushPhaseWaitingForSync) + }) + session := &archivepush.ReplicaSession{ + Connect: func(c context.Context) (storage.Pusher, error) { + return replica.NewPusher(c, target, local, storage.PusherOptions{ + Projects: body.Projects, + ExcludeProjects: body.ExcludeProjects, + SyncStateTarget: body.SyncStateTarget, + MigrateLegacySyncState: body.MigrateLegacySyncState, + VectorSource: vectorSource, + }) + }, + EnsurePricing: func(c context.Context) error { + return s.ensurePricing(c, local) + }, + } + if storage.StoresFriction(replica) { + session.FrictionReview = engine + } + var stage archivepush.Stage + session.OnStage = func(s archivepush.Stage) func() { + stage = s + return nil + } + defer session.Close() var result storage.PushResult err := s.syncThenRunForPush( - ctx, engine, local, body.Full, body.WatchBatch, body.WatchRecovery, + pushCtx, engine, local, body.Full, body.WatchBatch, body.WatchRecovery, + reportPhase, func(forceFull bool) error { - if storage.StoresFriction(replica) { - if err := engine.DrainStaleFrictionLocked(ctx); err != nil { - return err - } - } - if refreshErr := s.ensurePricing(ctx, local); refreshErr != nil { - if ctxErr := ctx.Err(); ctxErr != nil { - return ctxErr - } - log.Printf("pricing refresh: %v", refreshErr) - } - if ctxErr := ctx.Err(); ctxErr != nil { - return ctxErr - } - pusher, err := replica.NewPusher( - ctx, target, local, storage.PusherOptions{ - Projects: body.Projects, - ExcludeProjects: body.ExcludeProjects, - SyncStateTarget: body.SyncStateTarget, - MigrateLegacySyncState: body.MigrateLegacySyncState, - VectorSource: vectorSource, - }, - ) - if err != nil { - return err - } - defer pusher.Close() - if err := pusher.EnsureSchema(ctx); err != nil { - return err - } - result, err = pusher.PushWithOptions(ctx, storage.PushOptions{ + var err error + result, err = session.Push(pushCtx, storage.PushOptions{ Full: forceFull, ScopeVectorsToChangedSessions: body. ScopeVectorsToChangedSessions, LastReconciledVectorGeneration: body. LastReconciledVectorGeneration, }, onProgress) + // The replica records a full push only once the copy starts. + if err != nil && stage != archivepush.StagePush { + return storage.PushNotStartedError{Err: err} + } return err }, ) @@ -670,23 +824,40 @@ func (s *Server) humaMirrorPush( } engine := s.syncEngineForLocal(ctx, local) - opts := mirrorPushOptions(in.Body) + push := archivepush.MirrorPush{ + Mirror: s.mirror, Config: duckCfg, Archive: local, + Options: mirrorPushOptions(in.Body), + EnsurePricing: func(c context.Context) error { + return s.ensurePricing(c, local) + }, + } body := in.Body name := s.mirror.Name() return &huma.StreamResponse{Body: func(hctx huma.Context) { runPushStream(hctx, func( - streamProgress func(storage.MirrorPushProgress), + streamProgress, streamPhase func(storage.MirrorPushProgress), ) (any, error) { onProgress := composePushProgress( newMirrorPushProgressLogger(name), streamProgress, ) var result storage.MirrorPushResult - err := s.syncThenRunForPush(ctx, engine, local, body.Full, nil, nil, + var rebuilt bool + reportPhase := func(phase string) { + rebuilt = rebuilt || phase == storage.PushPhaseRebuilt + logPushPhase(name, phase) + if streamPhase != nil { + streamPhase(storage.MirrorPushProgress{Phase: phase}) + } + } + pushCtx := db.WithWriterWaitNotice(ctx, func() { + reportPhase(storage.PushPhaseWaitingForSync) + }) + err := s.syncThenRunForPush(pushCtx, engine, local, body.Full, nil, nil, + reportPhase, func(forceFull bool) error { var pushErr error - result, pushErr = s.mirror.Push( - ctx, duckCfg, local, opts, forceFull, onProgress, - ) + result, pushErr = push.Push(pushCtx, forceFull, onProgress) + result.ArchiveRebuilt = rebuilt return pushErr }) return result, err diff --git a/internal/server/huma_routes_push_internal_test.go b/internal/server/huma_routes_push_internal_test.go index 5bcd7fe29a..332deef5f6 100644 --- a/internal/server/huma_routes_push_internal_test.go +++ b/internal/server/huma_routes_push_internal_test.go @@ -5,12 +5,14 @@ import ( "context" "encoding/json/v2" "errors" + "fmt" "log" "net/http" "net/http/httptest" "os" "path/filepath" "strings" + "sync" "testing" "time" @@ -582,7 +584,7 @@ func TestSyncThenRunForPushWorkerRunnerRouting(t *testing.T) { t.Cleanup(engine.Close) err := f.srv.syncThenRunForPush( - t.Context(), engine, f.db, tt.full, nil, nil, + t.Context(), engine, f.db, tt.full, nil, nil, nil, func(forceFull bool) error { workCalls++ assert.Equal(t, tt.wantForceFull, forceFull) @@ -611,7 +613,7 @@ func TestSyncThenRunForPushRunnerErrorSkipsPush(t *testing.T) { t.Cleanup(engine.Close) err := f.srv.syncThenRunForPush( - t.Context(), engine, f.db, true, nil, nil, + t.Context(), engine, f.db, true, nil, nil, nil, func(bool) error { require.FailNow(t, "push work must not run after a failed worker pass") return nil @@ -644,9 +646,9 @@ func TestSyncThenRunForPushCopiesHealthyArchiveBesideCorruptSource(t *testing.T) request := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/api/v1/push/pg", nil) request.Header.Set("Accept", accept) hctx := humago.NewContext(&huma.Operation{}, request, recorder) - runPushStream(hctx, func(_ func(storage.PushProgress)) (any, error) { + runPushStream(hctx, func(_, _ func(storage.PushProgress)) (any, error) { var result storage.PushResult - err := f.srv.syncThenRunForPush(t.Context(), engine, f.db, false, nil, nil, + err := f.srv.syncThenRunForPush(t.Context(), engine, f.db, false, nil, nil, nil, func(forceFull bool) error { assert.False(t, forceFull) session, err := f.db.GetSession(t.Context(), "healthy") @@ -676,9 +678,111 @@ func TestSyncThenRunForPushCopiesHealthyArchiveBesideCorruptSource(t *testing.T) }) } pushFailure := errors.New("mirror write failed") - err := f.srv.syncThenRunForPush(t.Context(), engine, f.db, false, nil, nil, + err := f.srv.syncThenRunForPush(t.Context(), engine, f.db, false, nil, nil, nil, func(bool) error { return pushFailure }) require.ErrorIs(t, err, pushFailure) + _, notStarted := errors.AsType[storage.PushNotStartedError](err) + assert.False(t, notStarted, "an error from the push work may follow a started copy") +} + +// A daemon push must not block local ingestion: a sync can take the engine +// lock while the push copies the archive. +func TestSyncThenRunForPushLeavesSyncLockFreeDuringPush(t *testing.T) { + f := newSyncRouteFixture(t) + engine := f.srv.syncEngineForLocal(t.Context(), f.db) + t.Cleanup(engine.Close) + + err := f.srv.syncThenRunForPush(t.Context(), engine, f.db, false, nil, nil, nil, + func(bool) error { + return engine.TryRunExclusive(func() error { return nil }) + }) + + require.NoError(t, err, "a sync must be able to run while the push runs") +} + +// Daemon pushes take turns, so an older push can never overwrite what a newer +// push to the same destination wrote. A push takes its turn before its own +// sync, so its archive rebuild never replaces the archive another push reads. +func TestSyncThenRunForPushRunsOnePushAtATime(t *testing.T) { + var mu sync.Mutex + var events []string + record := func(event string) { + mu.Lock() + defer mu.Unlock() + events = append(events, event) + } + f := newSyncRouteFixture(t, withLocalResyncRunner(func( + context.Context, func(syncpkg.Progress), + ) (syncpkg.SyncStats, error) { + record("second rebuild") + return syncpkg.SyncStats{}, nil + })) + engine := f.srv.syncEngineForLocal(t.Context(), f.db) + t.Cleanup(engine.Close) + + firstStarted := make(chan struct{}) + releaseFirst := make(chan struct{}) + first := make(chan error, 1) + go func() { + first <- f.srv.syncThenRunForPush(t.Context(), engine, f.db, false, nil, nil, nil, + func(bool) error { + close(firstStarted) + <-releaseFirst + record("first done") + return nil + }) + }() + <-firstStarted + waiting := make(chan struct{}) + second := make(chan error, 1) + go func() { + second <- f.srv.syncThenRunForPush(t.Context(), engine, f.db, true, nil, nil, + func(phase string) { + if phase == storage.PushPhaseWaiting { + close(waiting) + } + }, + func(bool) error { + record("second start") + return nil + }) + }() + + <-waiting + close(releaseFirst) + require.NoError(t, <-first) + require.NoError(t, <-second) + assert.Equal(t, []string{"first done", "second rebuild", "second start"}, events) +} + +// A push waiting for its turn stops when its request ends. +func TestSyncThenRunForPushStopsWaitingForTurnWhenContextEnds(t *testing.T) { + f := newSyncRouteFixture(t) + engine := f.srv.syncEngineForLocal(t.Context(), f.db) + t.Cleanup(engine.Close) + endTurn, err := f.srv.pushTurn.take(t.Context(), nil) + require.NoError(t, err) + t.Cleanup(endTurn) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + waiting := make(chan struct{}) + done := make(chan error, 1) + go func() { + done <- f.srv.syncThenRunForPush(ctx, engine, f.db, false, nil, nil, + func(phase string) { + if phase == storage.PushPhaseWaiting { + close(waiting) + } + }, + func(bool) error { + assert.Fail(t, "a push must not run while another push holds the turn") + return nil + }) + }() + <-waiting + cancel() + require.ErrorIs(t, <-done, context.Canceled) } func TestSyncThenRunForPushDeferredWorkerSkipsPush(t *testing.T) { @@ -691,7 +795,7 @@ func TestSyncThenRunForPushDeferredWorkerSkipsPush(t *testing.T) { t.Cleanup(engine.Close) err := f.srv.syncThenRunForPush( - t.Context(), engine, f.db, true, nil, nil, + t.Context(), engine, f.db, true, nil, nil, nil, func(bool) error { require.FailNow(t, "push work must not run after deferred worker processing") return nil @@ -700,6 +804,8 @@ func TestSyncThenRunForPushDeferredWorkerSkipsPush(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "local sync processing incomplete") + _, notStarted := errors.AsType[storage.PushNotStartedError](err) + assert.True(t, notStarted, "a failed pre-push sync never started the push") } func TestSyncThenRunForPushAppliesCurrentWatchBatchBeforeWork(t *testing.T) { @@ -711,7 +817,7 @@ func TestSyncThenRunForPushAppliesCurrentWatchBatchBeforeWork(t *testing.T) { batch := syncpkg.WatchBatch{Paths: []string{changed}} err := f.srv.syncThenRunForPush( - t.Context(), engine, f.db, false, &batch, nil, + t.Context(), engine, f.db, false, &batch, nil, nil, func(forceFull bool) error { assert.False(t, forceFull) changedSession, getErr := f.db.GetSession(t.Context(), "changed") @@ -746,7 +852,7 @@ func TestSyncThenRunForPushStaleArchiveIgnoresBatchAndUsesWorker(t *testing.T) { workCalls := 0 err := f.srv.syncThenRunForPush( - t.Context(), engine, f.db, false, &batch, nil, + t.Context(), engine, f.db, false, &batch, nil, nil, func(forceFull bool) error { workCalls++ assert.True(t, forceFull) @@ -1043,3 +1149,233 @@ func TestClickHousePushRejectsPlaintextTargetBeforeStream(t *testing.T) { assert.Equal(t, http.StatusBadRequest, statusErr.GetStatus()) assert.Contains(t, err.Error(), "allow_insecure") } + +// A daemon push that waits for another push tells its client, so the CLI can +// say why nothing is happening. +func TestMirrorPushStreamsWaitingForAnotherPush(t *testing.T) { + f := newSyncRouteFixture(t) + f.srv.cfg.DuckDB = config.DuckDBConfig{ + Path: filepath.Join(t.TempDir(), "mirror.duckdb"), + MachineName: "daemon", + } + endTurn, err := f.srv.pushTurn.take(t.Context(), nil) + require.NoError(t, err) + t.Cleanup(endTurn) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + stream, err := f.srv.humaMirrorPush(ctx, &daemonPushInput{}) + require.NoError(t, err) + writer := newSignalingRecorder(`"Phase":"waiting"`) + request := httptest.NewRequestWithContext(ctx, http.MethodPost, "/api/v1/push/duckdb", nil) + request.Header.Set("Accept", "text/event-stream") + done := make(chan struct{}) + go func() { + defer close(done) + stream.Body(humago.NewContext(&huma.Operation{}, request, writer)) + }() + + <-writer.matched + cancel() + <-done + assert.Contains(t, writer.String(), "event: progress\n") +} + +// A daemon push whose sync rebuilt the archive reports it in the stream before +// the result and marks the mirror result. A push that rebuilt nothing reports +// neither, so a client whose stream drops keeps its --full request. +func TestMirrorPushReportsArchiveRebuild(t *testing.T) { + for _, rebuild := range []bool{false, true} { + t.Run(fmt.Sprintf("rebuild=%t", rebuild), func(t *testing.T) { + var f *syncRouteFixture + f = newSyncRouteFixture(t, withLocalResyncRunner(func( + ctx context.Context, _ func(syncpkg.Progress), + ) (syncpkg.SyncStats, error) { + return syncpkg.SyncStats{}, f.db.SetDatabaseIDForTest(ctx, "rebuilt") + })) + f.srv.cfg.DuckDB = config.DuckDBConfig{ + Path: filepath.Join(t.TempDir(), "mirror.duckdb"), + MachineName: "daemon", + } + // Only a full push routes its sync through the rebuilding runner. + stream, err := f.srv.humaMirrorPush(t.Context(), + &daemonPushInput{Body: daemonPushRequest{Full: rebuild}}) + require.NoError(t, err) + recorder := httptest.NewRecorder() + request := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/api/v1/push/duckdb", nil) + request.Header.Set("Accept", "text/event-stream") + stream.Body(humago.NewContext(&huma.Operation{}, request, recorder)) + + body := recorder.Body.String() + reported := strings.Index(body, `"Phase":"rebuilt"`) + finished := strings.Index(body, "event: done\n") + require.NotEqual(t, -1, finished, "push did not finish: %s", body) + var result storage.MirrorPushResult + require.NoError(t, json.Unmarshal( + []byte(strings.TrimSpace(strings.TrimPrefix(body[finished:], "event: done\ndata: "))), + &result, + )) + assert.Equal(t, rebuild, result.ArchiveRebuilt) + if !rebuild { + assert.Equal(t, -1, reported, "stream reports a rebuild: %s", body) + return + } + require.NotEqual(t, -1, reported, "stream has no rebuild report: %s", body) + assert.Less(t, reported, finished) + }) + } +} + +// signalingRecorder is a goroutine-safe flushing ResponseWriter that closes +// matched once its body contains want. +type signalingRecorder struct { + *httptest.ResponseRecorder + mu sync.Mutex + want string + matched chan struct{} + once sync.Once +} + +func newSignalingRecorder(want string) *signalingRecorder { + return &signalingRecorder{ + ResponseRecorder: httptest.NewRecorder(), + want: want, + matched: make(chan struct{}), + } +} + +func (r *signalingRecorder) Write(p []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + n, err := r.ResponseRecorder.Write(p) + if strings.Contains(r.Body.String(), r.want) { + r.once.Do(func() { close(r.matched) }) + } + return n, err +} + +func (r *signalingRecorder) Flush() { + r.mu.Lock() + defer r.mu.Unlock() + r.ResponseRecorder.Flush() +} + +func (r *signalingRecorder) String() string { + r.mu.Lock() + defer r.mu.Unlock() + return r.Body.String() +} + +// A push error tells the client whether the copy started, in both response +// shapes, so a watch can keep a --full request that no push started. +func TestRunPushStreamReportsWhetherThePushStarted(t *testing.T) { + for _, accept := range []string{"application/json", "text/event-stream"} { + for _, tt := range []struct { + err error + started bool + }{ + {storage.PushNotStartedError{Err: errors.New("resync failed")}, false}, + {errors.New("copy failed"), true}, + } { + t.Run(fmt.Sprintf("%s started=%t", accept, tt.started), func(t *testing.T) { + recorder := httptest.NewRecorder() + request := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/api/v1/push/pg", nil) + request.Header.Set("Accept", accept) + runPushStream(humago.NewContext(&huma.Operation{}, request, recorder), + func(_, _ func(storage.PushProgress)) (any, error) { return nil, tt.err }) + payload := recorder.Body.String() + if accept == "text/event-stream" { + require.Contains(t, payload, "event: error\n") + _, payload, _ = strings.Cut(payload, "data: ") + } + var body pushErrorResponse + require.NoError(t, json.Unmarshal([]byte(strings.TrimSpace(payload)), &body)) + assert.Equal(t, tt.err.Error(), body.Error) + assert.Equal(t, tt.started, body.PushStarted) + }) + } + } +} + +// stubReplica is a PostgreSQL replica whose pusher the test supplies. +type stubReplica struct { + postgres.Backend + newPusher func() (storage.Pusher, error) +} + +func (r stubReplica) NewPusher( + context.Context, storage.ReplicaTarget, *db.DB, storage.PusherOptions, +) (storage.Pusher, error) { + return r.newPusher() +} + +type failingPusher struct{ err error } + +func (failingPusher) EnsureSchema(context.Context) error { return nil } +func (p failingPusher) PushWithOptions( + context.Context, storage.PushOptions, func(storage.PushProgress), +) (storage.PushResult, error) { + return storage.PushResult{}, p.err +} +func (failingPusher) Close() error { return nil } + +// The replica route reports whether the copy started: a failed connect +// started nothing, and a failed copy may have recorded a full push the mirror +// resumes. +func TestReplicaPushReportsWhetherTheCopyStarted(t *testing.T) { + f := newSyncRouteFixture(t) + for _, tt := range []struct { + name string + newPusher func() (storage.Pusher, error) + started bool + }{ + {"connect failed", func() (storage.Pusher, error) { + return nil, errors.New("connection refused") + }, false}, + {"copy failed", func() (storage.Pusher, error) { + return failingPusher{errors.New("conn reset")}, nil + }, true}, + } { + t.Run(tt.name, func(t *testing.T) { + stream, err := f.srv.humaReplicaPush(t.Context(), + stubReplica{newPusher: tt.newPusher}, + &daemonPushInput{Body: daemonPushRequest{Replica: &daemonReplicaTarget{ + URL: "postgres://127.0.0.1:5432/agentsview?sslmode=disable", MachineName: "test", + }}}) + require.NoError(t, err) + recorder := httptest.NewRecorder() + request := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/api/v1/push/pg", nil) + request.Header.Set("Accept", "text/event-stream") + stream.Body(humago.NewContext(&huma.Operation{}, request, recorder)) + + _, payload, found := strings.Cut(recorder.Body.String(), "event: error\ndata: ") + require.True(t, found, recorder.Body.String()) + var body pushErrorResponse + require.NoError(t, json.Unmarshal([]byte(strings.TrimSpace(payload)), &body)) + assert.Equal(t, tt.started, body.PushStarted) + }) + } +} + +// A push that rebuilt the archive and then failed to connect reports the push +// as started. Every mirror pushes in full after a rebuild, and a client that +// kept --full would make the daemon rebuild the archive on every retry. +func TestSyncThenRunForPushCountsARebuildAsStarted(t *testing.T) { + var f *syncRouteFixture + f = newSyncRouteFixture(t, withLocalResyncRunner(func( + ctx context.Context, _ func(syncpkg.Progress), + ) (syncpkg.SyncStats, error) { + return syncpkg.SyncStats{}, f.db.SetDatabaseIDForTest(ctx, "rebuilt") + })) + engine := f.srv.syncEngineForLocal(t.Context(), f.db) + t.Cleanup(engine.Close) + connectErr := storage.PushNotStartedError{Err: errors.New("connection refused")} + + for _, full := range []bool{false, true} { + err := f.srv.syncThenRunForPush(t.Context(), engine, f.db, full, nil, nil, nil, + func(bool) error { return connectErr }) + require.ErrorIs(t, err, connectErr.Err) + assert.Equal(t, full, pushErrorBody(err.Error(), err).PushStarted, + "full=%t: only a push that rebuilt the archive counts as started", full) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 32ec7187eb..a38f0a3f0c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -93,6 +93,8 @@ type Server struct { settingsApplyMu gosync.Mutex // onDemandReconfigureMu serializes source updates to onDemandEngine. onDemandReconfigureMu gosync.Mutex + // pushTurn lets one archive push run at a time. + pushTurn pushTurn db db.Store activityReports *activityReportCache assetCache *assetCache diff --git a/internal/storage/mirror.go b/internal/storage/mirror.go index bfc75e5896..05e9a6e6dd 100644 --- a/internal/storage/mirror.go +++ b/internal/storage/mirror.go @@ -50,6 +50,10 @@ type MirrorPushResult struct { Errors int Duration time.Duration Diagnostics MirrorPushDiagnostics + // ArchiveRebuilt reports that a daemon push's sync rebuilt the archive + // before the push ran. The next push of every mirror then runs in full, + // so a deferred push need not keep a --full request. + ArchiveRebuilt bool } type mirrorPushResultJSON MirrorPushResult @@ -107,6 +111,11 @@ type MirrorSessionCounts struct { // MirrorPushProgress is reported after each attempted session. type MirrorPushProgress struct { + // Phase is PushPhaseWaiting while a daemon push waits for another push + // to finish, PushPhaseRebuilt once when its sync rebuilt the archive, + // PushPhaseWaitingForSync while it waits for a local sync to finish, and + // "" otherwise. + Phase string SessionsDone int SessionsTotal int MessagesDone int diff --git a/internal/storage/push.go b/internal/storage/push.go index d249b26d77..81e9c6107f 100644 --- a/internal/storage/push.go +++ b/internal/storage/push.go @@ -43,6 +43,9 @@ type PushProgress struct { // a full push this covers every local session and can run for minutes), // "" during the session/message push, and "vectors" during the vector // phase, whose progress is carried by the Vector* fields. + // PushPhaseWaiting means a daemon push is waiting for another push to + // finish, PushPhaseRebuilt that its sync rebuilt the archive, and + // PushPhaseWaitingForSync that it is waiting for a local sync to finish. Phase string SessionsDone int SessionsTotal int @@ -154,3 +157,26 @@ func (r *VectorPushResult) DeferSession(sessionID string) { r.SessionsDeferred++ r.DeferredSessionIDs = append(r.DeferredSessionIDs, sessionID) } + +// PushPhaseWaiting is the progress phase a daemon push reports while it +// waits for another push to finish. The daemon runs one push at a time. +const PushPhaseWaiting = "waiting" + +// PushNotStartedError marks a push error from before the push recorded a +// full push it could resume. A watch keeps its --full request after one; any +// other failed push resumes its recorded full push instead. +type PushNotStartedError struct{ Err error } + +func (e PushNotStartedError) Error() string { return e.Err.Error() } +func (e PushNotStartedError) Unwrap() error { return e.Err } + +// PushPhaseRebuilt is the progress phase a daemon push reports as soon as its +// sync has rebuilt the archive. Every mirror pushes in full after a rebuild, +// so a client whose stream ends after this report need not ask for --full +// again. A client whose stream ends without it keeps its --full request. +const PushPhaseRebuilt = "rebuilt" + +// PushPhaseWaitingForSync is the progress phase a daemon push reports when it +// has waited a few seconds for a local sync to hand the archive writer back. +// The push resumes once the sync finishes. +const PushPhaseWaitingForSync = "waiting_for_sync" diff --git a/internal/sync/engine.go b/internal/sync/engine.go index e0f7eec563..93e1683251 100644 --- a/internal/sync/engine.go +++ b/internal/sync/engine.go @@ -5035,6 +5035,39 @@ func (e *Engine) RunExclusiveFlushed(work func() error) error { return work() } +// SettleForPush waits for an in-progress sync to finish and then flushes +// deferred signal recomputes, so a push that starts next copies settled +// sessions. It does not hold the sync lock afterward: later syncs may write +// while the push runs. If ctx ends first, SettleForPush returns ctx.Err() and +// abandons the wait; the lock is released without flushing once acquired. +func (e *Engine) SettleForPush(ctx context.Context) error { + if e.refuseWriteInForceParse("SettleForPush") { + return errors.New( + "SettleForPush refused on report-only parse-diff engine", + ) + } + if err := ctx.Err(); err != nil { + return err + } + var abandoned atomic.Bool + settled := make(chan struct{}) + go func() { + defer close(settled) + e.syncMu.Lock() + defer e.syncMu.Unlock() + if !abandoned.Load() { + e.signalSched.flushAllInline() + } + }() + select { + case <-settled: + return nil + case <-ctx.Done(): + abandoned.Store(true) + return ctx.Err() + } +} + // ApplyWorktreeReclassification serializes the mapping rule, historical // session rewrites, and identity publication with watcher and sync writes. func (e *Engine) ApplyWorktreeReclassification( diff --git a/internal/sync/progress.go b/internal/sync/progress.go index 581bf19d00..c3dc1b0c14 100644 --- a/internal/sync/progress.go +++ b/internal/sync/progress.go @@ -145,6 +145,12 @@ func (s *SyncStats) shouldEmitSync() bool { (!s.Aborted && (s.hasSessionChanges() || s.ArchiveRebuilt)) } +// ChangedArchive reports whether the pass wrote session state to the archive, +// including a pass that stopped early after some writes. +func (s *SyncStats) ChangedArchive() bool { + return s.hasSessionChanges() || s.ArchiveRebuilt +} + func (s *SyncStats) hasSessionChanges() bool { return s.Synced > 0 || s.CwdUpdated > 0 || s.TitlesUpdated > 0 || s.Tombstoned > 0 || s.LinksUpdated > 0 } diff --git a/internal/sync/signal_schedule_test.go b/internal/sync/signal_schedule_test.go index acdfe590ac..af1a54db65 100644 --- a/internal/sync/signal_schedule_test.go +++ b/internal/sync/signal_schedule_test.go @@ -1,6 +1,7 @@ package sync import ( + "context" "os" "sync" "testing" @@ -10,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.kenn.io/agentsview/internal/testjsonl" + "go.kenn.io/agentsview/internal/testutil" ) type harnessTimer struct { @@ -620,6 +622,59 @@ func TestRunExclusiveFlushedFlushesSignalsBeforeWork(t *testing.T) { "work must observe flushed signal fields before pushing") } +func TestSettleForPushFlushesSignals(t *testing.T) { + fx := newEngineFixture(t) + path := fx.writeClaudeSession(t, "proj", "sig-flush-settle.jsonl", "hello") + fx.engine.SyncAll(t.Context(), nil) + sid := fx.sessionIDFor(t, path) + + fx.appendClaudeMessage(t, path, "key AKIA7QHWN2DKR4FYPLJM leaked") + fx.engine.SyncPaths([]string{path}) + fx.appendClaudeMessage(t, path, "key AKIA9XKQV3ZTN8WMB2RC leaked") + fx.engine.SyncPaths([]string{path}) + require.Equal(t, 1, secretLeakCount(t, fx, sid), + "second write within interval should defer the recompute") + + require.NoError(t, fx.engine.SettleForPush(t.Context())) + assert.Equal(t, 2, secretLeakCount(t, fx, sid), + "a push after settling must see flushed signal fields") +} + +// A shutdown flush must not wait out a sync that holds the engine lock. +func TestSettleForPushStopsWaitingWhenContextEnds(t *testing.T) { + fx := newEngineFixture(t) + held := make(chan struct{}) + release := make(chan struct{}) + releaseLock := sync.OnceFunc(func() { close(release) }) + t.Cleanup(releaseLock) + exclusive := make(chan error, 1) + go func() { + exclusive <- fx.engine.RunExclusive(func() error { + close(held) + <-release + return nil + }) + }() + <-held + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + settled := make(chan error, 1) + go func() { settled <- fx.engine.SettleForPush(ctx) }() + // Cancel only once SettleForPush waits for the lock, so the test covers + // abandoning a wait and not the context check before it. + require.Eventually(t, func() bool { + return testutil.GoroutineBlockedOnMutexIn("(*Engine).SettleForPush") + }, 10*time.Second, 5*time.Millisecond) + cancel() + require.ErrorIs(t, <-settled, context.Canceled) + + releaseLock() + require.NoError(t, <-exclusive) + require.NoError(t, fx.engine.SettleForPush(t.Context()), + "the abandoned wait must release the lock") +} + func TestSignalSchedulerRealTimerFlushes(t *testing.T) { synctest.Test(t, func(t *testing.T) { var mu sync.Mutex diff --git a/internal/testutil/goroutine.go b/internal/testutil/goroutine.go new file mode 100644 index 0000000000..7407892a16 --- /dev/null +++ b/internal/testutil/goroutine.go @@ -0,0 +1,22 @@ +package testutil + +import ( + "runtime" + "strings" +) + +// GoroutineBlockedOnMutexIn reports whether some goroutine is waiting to lock +// a sync.Mutex inside a function whose name contains fn. Go has no API for +// that, and testing/synctest does not treat a mutex wait as blocked, so this +// reads the text of every goroutine's stack. Tests use it to act only once +// another goroutine is parked on a lock. +func GoroutineBlockedOnMutexIn(fn string) bool { + buf := make([]byte, 1<<20) + for g := range strings.SplitSeq(string(buf[:runtime.Stack(buf, true)]), "\n\n") { + header, _, _ := strings.Cut(g, "\n") + if strings.Contains(header, "[sync.Mutex.Lock") && strings.Contains(g, fn) { + return true + } + } + return false +} From 111b8c9d91b9fb908fcd89d0652964bd386c8b99 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 19 Oct 2026 07:52:10 -0400 Subject: [PATCH 02/11] fix(push): keep Codex thread curation and let a canceled push stop waiting After a rebuild, a full PostgreSQL push deletes sessions the archive no longer has. An old Codex thread row can be one of them: the rebuilt archive stores its transcript as a page, and the thread row still holds the user's pins, star, and title. A full push normally publishes the page and moves that curation onto it first. A push filtered to the thread's project skips a page in another project, so nothing moved the curation and the cleanup deleted it. The cleanup now runs the same page migration the project-move cleanup uses before it deletes, so an excluded page fails the push with the existing error that asks to include its project, and the thread row stays. A canceled daemon push could still wait for a long archive write before it took its cutoff. That push kept the daemon's one-push turn, which blocked other pushes and could overrun the shutdown deadline. The cutoff now stops waiting when its request ends. It still queues for the lock normally, so a stream of writes cannot starve it. Co-Authored-By: Claude Opus 5.5 --- cmd/agentsview/archive_write_backend.go | 4 +- internal/db/sessions.go | 28 ++++++- internal/db/sessions_sync_marker_test.go | 41 +++++++++- .../codex_page_retirement_pgtest_test.go | 74 +++++++++++++++++++ internal/postgres/push.go | 46 ++++++++++-- internal/postgres/push_pgtest_test.go | 2 +- 6 files changed, 181 insertions(+), 14 deletions(-) diff --git a/cmd/agentsview/archive_write_backend.go b/cmd/agentsview/archive_write_backend.go index 9022390f51..7dff508d30 100644 --- a/cmd/agentsview/archive_write_backend.go +++ b/cmd/agentsview/archive_write_backend.go @@ -452,8 +452,8 @@ func startDaemonPushHeartbeatTo(w io.Writer, label string) func() { // one line saying so and keeps the heartbeat. The report that the archive // was rebuilt also keeps it, since schema migration may follow without // progress events. A push that waits for a local sync prints a line each time -// the daemon reports the wait. The returned finish func stops the heartbeat (if no event ever -// arrived) and clears the in-place line. +// the daemon reports the wait. The returned finish func stops the heartbeat +// (if no event ever arrived) and clears the in-place line. func daemonPushProgress[P any]( label string, phase func(P) string, render func(P), ) (onProgress func(P), finish func()) { diff --git a/internal/db/sessions.go b/internal/db/sessions.go index 8441b199a4..90719295cf 100644 --- a/internal/db/sessions.go +++ b/internal/db/sessions.go @@ -6808,13 +6808,15 @@ func (db *DB) CountSessionsForMirrorScope( // It also waits while the writer is closed for a worker handoff. A sync-worker // child writes the archive then, outside this process's write lock, so a // cutoff taken during the handoff has the same gap. MirrorPushCutoff returns -// ctx.Err() if ctx ends first. +// ctx.Err() if ctx ends first, including while a long write holds the lock. func (db *DB) MirrorPushCutoff(ctx context.Context) (string, error) { for { if err := db.WaitForWriter(ctx); err != nil { return "", err } - db.mu.Lock() + if err := db.lockWriterCtx(ctx); err != nil { + return "", err + } if !db.writerClosed.Load() { cutoff := time.Now().UTC().Format(mirrorWindowBoundLayout) db.mu.Unlock() @@ -6824,6 +6826,28 @@ func (db *DB) MirrorPushCutoff(ctx context.Context) (string, error) { } } +// lockWriterCtx takes the write lock in the lock's normal queue, so a cutoff +// is not starved by back-to-back writes, and returns if ctx ends first. A +// request that gives up leaves a goroutine that releases the lock once it is +// granted. +func (db *DB) lockWriterCtx(ctx context.Context) error { + locked := make(chan struct{}) + go func() { + db.mu.Lock() + close(locked) + }() + select { + case <-locked: + return nil + case <-ctx.Done(): + go func() { + <-locked + db.mu.Unlock() + }() + return fmt.Errorf("stopped while waiting for an archive write to finish: %w", ctx.Err()) + } +} + // WaitForWriter blocks while the writer is closed for a worker handoff. It // returns ctx.Err() if ctx ends first. A write can still fail if a new // handoff starts after it returns. A wait that lasts writerHandoffNoticeAfter diff --git a/internal/db/sessions_sync_marker_test.go b/internal/db/sessions_sync_marker_test.go index 14971453cb..c7f0979f37 100644 --- a/internal/db/sessions_sync_marker_test.go +++ b/internal/db/sessions_sync_marker_test.go @@ -7,6 +7,7 @@ import ( "database/sql" "encoding/json" "runtime" + "sync" "sync/atomic" "testing" "testing/synctest" @@ -537,7 +538,7 @@ func TestMirrorPushCutoffCoversOpenWriteTransaction(t *testing.T) { // lock, so a push that ignores the open transaction lists first. var got listing received := false - for !received && !testutil.GoroutineBlockedOnMutexIn("(*DB).MirrorPushCutoff") { + for !received && !testutil.GoroutineBlockedOnMutexIn("(*DB).lockWriterCtx") { select { case got = <-listed: received = true @@ -667,3 +668,41 @@ func TestWaitForWriterSaysItGaveUpOnATimeout(t *testing.T) { "gave up waiting for the local sync to finish: context deadline exceeded") }) } + +// A canceled push must not wait out a long archive write for its cutoff: it +// would keep the daemon's push turn and hold up shutdown. +func TestMirrorPushCutoffStopsWaitingBehindALongWrite(t *testing.T) { + database := testDB(t) + inWrite := make(chan struct{}) + release := make(chan struct{}) + // Release the write before the database closes, even when the test fails. + endWrite := sync.OnceFunc(func() { close(release) }) + t.Cleanup(endWrite) + writeDone := make(chan error, 1) + go func() { + writeDone <- database.Update(t.Context(), func(*sql.Tx) error { + close(inWrite) + <-release + return nil + }) + }() + <-inWrite + ctx, cancel := context.WithCancel(t.Context()) + cutoff := make(chan error, 1) + go func() { + _, err := database.MirrorPushCutoff(ctx) + cutoff <- err + }() + cancel() + select { + case err := <-cutoff: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + require.Fail(t, "the cutoff waited for the write after its request ended") + } + endWrite() + require.NoError(t, <-writeDone) + // The abandoned lock request must not keep the lock. + _, err := database.MirrorPushCutoff(t.Context()) + require.NoError(t, err) +} diff --git a/internal/postgres/codex_page_retirement_pgtest_test.go b/internal/postgres/codex_page_retirement_pgtest_test.go index 672151f5de..388a0cf0e2 100644 --- a/internal/postgres/codex_page_retirement_pgtest_test.go +++ b/internal/postgres/codex_page_retirement_pgtest_test.go @@ -388,3 +388,77 @@ func TestCodexPageUpgradeIncrementalPushCopiesFriction(t *testing.T) { require.Len(t, got, 1) assert.Equal(t, "come back later", got[0].Label) } + +// A rebuild can turn a legacy Codex thread into a page in another project. A +// full push that filters that project out must not delete the thread row, +// which still holds the only copy of its pins, star, and title. +func TestFilteredFullPushKeepsCodexThreadWhosePageIsExcluded(t *testing.T) { + const thread = "codex:11111111-1111-4111-8111-111111111111" + const page = thread + "_22222222-2222-4222-8222-222222222222" + const schema = "agentsview_codex_page_filtered_cleanup_test" + ctx := t.Context() + pgURL := testPGURL(t) + cleanNamedPGSchema(t, pgURL, schema) + t.Cleanup(func() { cleanNamedPGSchema(t, pgURL, schema) }) + source := testDB(t) + sourcePath := source.Path() + pagePath := filepath.Join(t.TempDir(), "rollout-2026-09-25T12-00-00-"+page[len("codex:"):]+".jsonl") + sess := db.Session{ + ID: thread, Agent: "codex", Project: "sample", Machine: "machine", + FilePath: &pagePath, MessageCount: 1, CreatedAt: "2026-09-25T12:00:00Z", + } + require.NoError(t, source.UpsertSession(ctx, sess)) + message := db.Message{ + SessionID: thread, Role: "assistant", Content: "retained page answer", + Timestamp: "2026-09-25T12:00:00Z", + } + require.NoError(t, source.InsertMessages(ctx, []db.Message{message})) + require.NoError(t, source.SetSessionDataVersion(ctx, thread, 130)) + projects := storage.PusherOptions{Projects: []string{"sample"}} + syncer, err := New(pgURL, schema, source, "machine", true, projects) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, syncer.Close()) }) + pushed, err := syncer.Push(ctx, true, nil) + require.NoError(t, err) + require.Zero(t, pushed.Errors) + store := &Store{pg: syncer.pg} + require.NoError(t, store.RenameSession(ctx, thread, new("remote page title"))) + starred, err := store.StarSession(ctx, thread) + require.NoError(t, err) + require.True(t, starred) + _, err = store.PinMessage(ctx, thread, 0, new("remote note")) + require.NoError(t, err) + + require.NoError(t, source.Close()) + raw, err := sql.Open("sqlite3", sourcePath) + require.NoError(t, err) + _, err = raw.ExecContext(ctx, "PRAGMA user_version=130") + require.NoError(t, err) + require.NoError(t, raw.Close()) + rebuilt := testDB(t) + require.NoError(t, rebuilt.CopyArchiveIdentityFrom(sourcePath)) + require.NoError(t, rebuilt.CopySyncStateFrom(sourcePath)) + sess.ID = page + sess.Project = "other" + sess.ParentSessionID = new(thread) + sess.RelationshipType = "continuation" + require.NoError(t, rebuilt.UpsertSession(ctx, sess)) + message.SessionID = page + require.NoError(t, rebuilt.InsertMessages(ctx, []db.Message{message})) + require.NoError(t, rebuilt.SetSessionDataVersion(ctx, page, db.CurrentDataVersion())) + + upgraded, err := New(pgURL, schema, rebuilt, "machine", true, projects) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, upgraded.Close()) }) + _, err = upgraded.Push(ctx, true, nil) + require.ErrorContains(t, err, `requires including project "other"`) + + row, err := store.GetSessionFull(ctx, thread) + require.NoError(t, err) + require.NotNil(t, row, "the push deleted the thread row") + require.NotNil(t, row.DisplayName) + assert.Equal(t, "remote page title", *row.DisplayName) + pins, err := store.ListPinnedMessages(ctx, thread, "") + require.NoError(t, err) + assert.Len(t, pins, 1) +} diff --git a/internal/postgres/push.go b/internal/postgres/push.go index 11fdb03728..f01a3f4bb3 100644 --- a/internal/postgres/push.go +++ b/internal/postgres/push.go @@ -634,7 +634,7 @@ func (s *Sync) pushWithOptions( return result, err } if full && result.Errors == 0 { - if err := s.reconcilePGMissingSessions(ctx, markerID); err != nil { + if err := s.reconcilePGMissingSessions(ctx, markerID, legacyMarkerMachines); err != nil { return result, err } } @@ -766,7 +766,7 @@ func (s *Sync) pushWithOptions( return result, err } if full && result.Errors == 0 { - if err := s.reconcilePGMissingSessions(ctx, markerID); err != nil { + if err := s.reconcilePGMissingSessions(ctx, markerID, legacyMarkerMachines); err != nil { return result, err } } @@ -2097,7 +2097,9 @@ func purgePGExcludedPushSessions( // Reconcile after publishing sessions so Codex page curation transfers finish // before deletion cascades. The unfinished full run keeps failed cleanup retryable. -func (s *Sync) reconcilePGMissingSessions(ctx context.Context, ownerMarker string) error { +func (s *Sync) reconcilePGMissingSessions( + ctx context.Context, ownerMarker string, legacyMarkerMachines []string, +) error { // Empty Codex thread anchors still own remote restore/purge scope. where := "owner_marker = $1 AND NOT (trash_includes_codex_pages AND deleted_at IS NOT NULL)" args := []any{ownerMarker} @@ -2109,18 +2111,25 @@ func (s *Sync) reconcilePGMissingSessions(ctx context.Context, ownerMarker strin args = append(args, s.excludeProjects) where += fmt.Sprintf(" AND project <> ALL($%d)", len(args)) } - rows, err := s.pg.QueryContext(ctx, "SELECT id FROM sessions WHERE "+where, args...) + rows, err := s.pg.QueryContext(ctx, + "SELECT id, agent, COALESCE(file_path, '') FROM sessions WHERE "+where, args...) if err != nil { return fmt.Errorf("listing owned pg sessions for full push reconciliation: %w", err) } defer rows.Close() var ids []string + // A legacy Codex thread row names the file of the page that replaced it. + codexPaths := make(map[string]string) for rows.Next() { - var id string - if err := rows.Scan(&id); err != nil { + var id, agent, path string + if err := rows.Scan(&id, &agent, &path); err != nil { return fmt.Errorf("scanning owned pg session for full push reconciliation: %w", err) } ids = append(ids, id) + switch agent { + case "codex", "traex", "augure-code": + codexPaths[id] = path + } } if err := rows.Err(); err != nil { return err @@ -2136,9 +2145,23 @@ func (s *Sync) reconcilePGMissingSessions(ctx context.Context, ownerMarker strin return err } var missing []string + var pages []db.Session for _, id := range chunk { - if _, exists := present[id]; !exists { - missing = append(missing, id) + if _, exists := present[id]; exists { + continue + } + missing = append(missing, id) + path, ok := codexPaths[id] + key := parser.CodexSessionUUIDFromPath(path) + if !ok || key == "" { + continue + } + page, err := s.local.GetArtifactExportSession(ctx, id[:strings.LastIndex(id, ":")+1]+key) + if err != nil { + return fmt.Errorf("reading retained Codex page: %w", err) + } + if page != nil { + pages = append(pages, *page) } } if len(missing) == 0 { @@ -2149,6 +2172,13 @@ func (s *Sync) reconcilePGMissingSessions(ctx context.Context, ownerMarker strin return err } defer func() { _ = tx.Rollback() }() + // A filtered push skips a page outside its projects, so the batch push + // never moved the old thread's pins, stars, and title onto it. Move + // them now, or fail when the page is excluded, before the delete + // cascades to that curation. + if err := s.migrateCodexPages(ctx, tx, pages, ownerMarker, legacyMarkerMachines); err != nil { + return err + } // Recheck ownership and scope, and clear vectors only for deleted rows. rows, err := tx.QueryContext(ctx, "DELETE FROM sessions WHERE "+where+fmt.Sprintf(" AND id = ANY($%d) RETURNING id", len(args)+1), diff --git a/internal/postgres/push_pgtest_test.go b/internal/postgres/push_pgtest_test.go index 9c43958031..189f737de9 100644 --- a/internal/postgres/push_pgtest_test.go +++ b/internal/postgres/push_pgtest_test.go @@ -3774,7 +3774,7 @@ func TestFilteredPushReconcilesProjectMoveCommittedDuringPush(t *testing.T) { // finished without waiting. var pushErr error finished := false - for !finished && !testutil.GoroutineBlockedOnMutexIn("(*DB).MirrorPushCutoff") { + for !finished && !testutil.GoroutineBlockedOnMutexIn("(*DB).lockWriterCtx") { select { case pushErr = <-pushed: finished = true From c85b2a80165d098fca98ddbaf35be16e7b739ebd Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 19 Oct 2026 08:54:36 -0400 Subject: [PATCH 03/11] fix(push): fail on excluded Codex pages only when curation is at stake The previous commit made a filtered full PostgreSQL push fail whenever an old Codex thread's rebuilt page sat in an excluded project. A thread with no pins, star, rename, or trash change has nothing to lose, so that failure only blocked the push and the cleanup of every other missing session. The cleanup now deletes such a thread as before and fails only when the thread holds curation. When the push does fail, the error said a project had to be included but not how. It now names the thread, says it was kept, and points to --projects, --exclude-projects, and the exclude_projects setting. A canceled daemon push could still hold the daemon's push turn in two places. Saving push progress and bookmarks waited for the archive write lock without checking for cancellation, so a long write outlasted the 30-second save limit. Friction review before a replica push took the sync lock the same way, so a canceled push waited for a whole sync that started after its settle step. Both waits now end when the push is canceled, and both still queue for their lock normally, so a busy writer cannot starve them. Co-Authored-By: Claude Opus 5.5 --- internal/archivepush/archivepush.go | 2 +- internal/db/db.go | 14 +- internal/db/sessions.go | 2 +- internal/db/sessions_sync_marker_test.go | 52 ++++++ internal/postgres/codex_page_pins.go | 5 +- .../codex_page_retirement_pgtest_test.go | 150 ++++++++++-------- internal/postgres/push.go | 33 +++- internal/sync/engine.go | 30 ++++ internal/sync/signal_schedule_test.go | 47 ++++++ 9 files changed, 257 insertions(+), 78 deletions(-) diff --git a/internal/archivepush/archivepush.go b/internal/archivepush/archivepush.go index f36c898d12..57a3bc0654 100644 --- a/internal/archivepush/archivepush.go +++ b/internal/archivepush/archivepush.go @@ -73,7 +73,7 @@ func (s *ReplicaSession) Push( ) (storage.PushResult, error) { if engine := s.FrictionReview; engine != nil { // The push itself runs without the sync lock. - if err := engine.RunExclusive(func() error { + if err := engine.RunExclusiveCtx(ctx, func() error { return engine.DrainStaleFrictionLocked(ctx) }); err != nil { return storage.PushResult{}, err diff --git a/internal/db/db.go b/internal/db/db.go index f7b04b9eb2..02505be2a4 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -5569,9 +5569,11 @@ func (db *DB) WriterClosed() bool { // Update executes fn within a write lock and transaction. // The transaction is committed if fn returns nil, rolled back -// otherwise. +// otherwise. It stops waiting for the lock when ctx ends. func (db *DB) Update(ctx context.Context, fn func(tx *sql.Tx) error) error { - db.mu.Lock() + if err := db.lockWriterCtx(ctx); err != nil { + return err + } defer db.mu.Unlock() // Fail fast before handing out a raw *sql.Tx: while the writer is closed @@ -5620,7 +5622,9 @@ func scanSyncState(row *sql.Row) (string, error) { // SetSyncState writes a value to the pg_sync_state table. func (db *DB) SetSyncState(ctx context.Context, key, value string) error { - db.mu.Lock() + if err := db.lockWriterCtx(ctx); err != nil { + return err + } defer db.mu.Unlock() _, err := db.getWriter().Exec(ctx, setSyncStateQuery, @@ -5729,7 +5733,9 @@ func escapeSyncStatePrefix(prefix string) string { // GetOrCreateSyncState returns a sync-state value, atomically creating it // with defaultValue when absent. func (db *DB) GetOrCreateSyncState(ctx context.Context, key, defaultValue string) (string, error) { - db.mu.Lock() + if err := db.lockWriterCtx(ctx); err != nil { + return "", err + } defer db.mu.Unlock() w := db.getWriter() var value string diff --git a/internal/db/sessions.go b/internal/db/sessions.go index 90719295cf..766f1f0eb7 100644 --- a/internal/db/sessions.go +++ b/internal/db/sessions.go @@ -6826,7 +6826,7 @@ func (db *DB) MirrorPushCutoff(ctx context.Context) (string, error) { } } -// lockWriterCtx takes the write lock in the lock's normal queue, so a cutoff +// lockWriterCtx takes the write lock in the lock's normal queue, so a caller // is not starved by back-to-back writes, and returns if ctx ends first. A // request that gives up leaves a goroutine that releases the lock once it is // granted. diff --git a/internal/db/sessions_sync_marker_test.go b/internal/db/sessions_sync_marker_test.go index c7f0979f37..467f18b81b 100644 --- a/internal/db/sessions_sync_marker_test.go +++ b/internal/db/sessions_sync_marker_test.go @@ -706,3 +706,55 @@ func TestMirrorPushCutoffStopsWaitingBehindALongWrite(t *testing.T) { _, err := database.MirrorPushCutoff(t.Context()) require.NoError(t, err) } + +// A push saves its progress with a time limit, and a canceled push must stop +// saving. Both must hold while a long archive write has the lock, or the push +// keeps the daemon's push turn until the write ends. +func TestPushStateWritesStopWaitingBehindALongWrite(t *testing.T) { + for name, write := range map[string]func(context.Context, *DB) error{ + "SetSyncState": func(ctx context.Context, d *DB) error { + return d.SetSyncState(ctx, "last_push_at", "cutoff") + }, + "GetOrCreateSyncState": func(ctx context.Context, d *DB) error { + _, err := d.GetOrCreateSyncState(ctx, "pg_push_marker_id", "marker") + return err + }, + "Update": func(ctx context.Context, d *DB) error { + return d.Update(ctx, func(*sql.Tx) error { return nil }) + }, + } { + t.Run(name, func(t *testing.T) { + database := testDB(t) + inWrite := make(chan struct{}) + release := make(chan struct{}) + endWrite := sync.OnceFunc(func() { close(release) }) + t.Cleanup(endWrite) + writeDone := make(chan error, 1) + go func() { + writeDone <- database.Update(t.Context(), func(*sql.Tx) error { + close(inWrite) + <-release + return nil + }) + }() + <-inWrite + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + saved := make(chan error, 1) + go func() { saved <- write(ctx, database) }() + require.Eventually(t, func() bool { + return testutil.GoroutineBlockedOnMutexIn("(*DB).lockWriterCtx") + }, 10*time.Second, 5*time.Millisecond) + cancel() + select { + case err := <-saved: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + require.Fail(t, "the canceled write kept waiting for the lock") + } + endWrite() + require.NoError(t, <-writeDone) + require.NoError(t, write(t.Context(), database)) + }) + } +} diff --git a/internal/postgres/codex_page_pins.go b/internal/postgres/codex_page_pins.go index 0d1bb655b2..2ff9aa0207 100644 --- a/internal/postgres/codex_page_pins.go +++ b/internal/postgres/codex_page_pins.go @@ -152,7 +152,10 @@ func (s *Sync) migrateCodexPages( return fmt.Errorf("cannot replace legacy Codex thread: retained page %s is unavailable", pageID) } if projectFailsFilter(page.Project, s.projects, s.excludeProjects) { - return fmt.Errorf("preserving the Codex page requires including project %q in the push", page.Project) + return fmt.Errorf( + "kept Codex thread %s: its pins and other curation move to its page in project %q, which this push excludes; "+ + "include that project with --projects, or remove it from --exclude-projects or the exclude_projects setting", + old.id, page.Project) } var pageExists bool if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM sessions WHERE id = $1)`, pageID).Scan(&pageExists); err != nil { diff --git a/internal/postgres/codex_page_retirement_pgtest_test.go b/internal/postgres/codex_page_retirement_pgtest_test.go index 388a0cf0e2..5f91f2eca6 100644 --- a/internal/postgres/codex_page_retirement_pgtest_test.go +++ b/internal/postgres/codex_page_retirement_pgtest_test.go @@ -390,75 +390,87 @@ func TestCodexPageUpgradeIncrementalPushCopiesFriction(t *testing.T) { } // A rebuild can turn a legacy Codex thread into a page in another project. A -// full push that filters that project out must not delete the thread row, -// which still holds the only copy of its pins, star, and title. -func TestFilteredFullPushKeepsCodexThreadWhosePageIsExcluded(t *testing.T) { - const thread = "codex:11111111-1111-4111-8111-111111111111" - const page = thread + "_22222222-2222-4222-8222-222222222222" - const schema = "agentsview_codex_page_filtered_cleanup_test" - ctx := t.Context() - pgURL := testPGURL(t) - cleanNamedPGSchema(t, pgURL, schema) - t.Cleanup(func() { cleanNamedPGSchema(t, pgURL, schema) }) - source := testDB(t) - sourcePath := source.Path() - pagePath := filepath.Join(t.TempDir(), "rollout-2026-09-25T12-00-00-"+page[len("codex:"):]+".jsonl") - sess := db.Session{ - ID: thread, Agent: "codex", Project: "sample", Machine: "machine", - FilePath: &pagePath, MessageCount: 1, CreatedAt: "2026-09-25T12:00:00Z", - } - require.NoError(t, source.UpsertSession(ctx, sess)) - message := db.Message{ - SessionID: thread, Role: "assistant", Content: "retained page answer", - Timestamp: "2026-09-25T12:00:00Z", - } - require.NoError(t, source.InsertMessages(ctx, []db.Message{message})) - require.NoError(t, source.SetSessionDataVersion(ctx, thread, 130)) - projects := storage.PusherOptions{Projects: []string{"sample"}} - syncer, err := New(pgURL, schema, source, "machine", true, projects) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, syncer.Close()) }) - pushed, err := syncer.Push(ctx, true, nil) - require.NoError(t, err) - require.Zero(t, pushed.Errors) - store := &Store{pg: syncer.pg} - require.NoError(t, store.RenameSession(ctx, thread, new("remote page title"))) - starred, err := store.StarSession(ctx, thread) - require.NoError(t, err) - require.True(t, starred) - _, err = store.PinMessage(ctx, thread, 0, new("remote note")) - require.NoError(t, err) - - require.NoError(t, source.Close()) - raw, err := sql.Open("sqlite3", sourcePath) - require.NoError(t, err) - _, err = raw.ExecContext(ctx, "PRAGMA user_version=130") - require.NoError(t, err) - require.NoError(t, raw.Close()) - rebuilt := testDB(t) - require.NoError(t, rebuilt.CopyArchiveIdentityFrom(sourcePath)) - require.NoError(t, rebuilt.CopySyncStateFrom(sourcePath)) - sess.ID = page - sess.Project = "other" - sess.ParentSessionID = new(thread) - sess.RelationshipType = "continuation" - require.NoError(t, rebuilt.UpsertSession(ctx, sess)) - message.SessionID = page - require.NoError(t, rebuilt.InsertMessages(ctx, []db.Message{message})) - require.NoError(t, rebuilt.SetSessionDataVersion(ctx, page, db.CurrentDataVersion())) +// full push that filters that project out must not delete a thread row that +// holds the only copy of its pins, star, and title. A thread with no curation +// has nothing to lose, so the push deletes it and succeeds. +func TestFilteredFullPushKeepsCuratedCodexThreadWhosePageIsExcluded(t *testing.T) { + for _, curated := range []bool{true, false} { + t.Run(fmt.Sprintf("curated=%t", curated), func(t *testing.T) { + const thread = "codex:11111111-1111-4111-8111-111111111111" + const page = thread + "_22222222-2222-4222-8222-222222222222" + const schema = "agentsview_codex_page_filtered_cleanup_test" + ctx := t.Context() + pgURL := testPGURL(t) + cleanNamedPGSchema(t, pgURL, schema) + t.Cleanup(func() { cleanNamedPGSchema(t, pgURL, schema) }) + source := testDB(t) + sourcePath := source.Path() + pagePath := filepath.Join(t.TempDir(), "rollout-2026-09-25T12-00-00-"+page[len("codex:"):]+".jsonl") + sess := db.Session{ + ID: thread, Agent: "codex", Project: "sample", Machine: "machine", + FilePath: &pagePath, MessageCount: 1, CreatedAt: "2026-09-25T12:00:00Z", + } + require.NoError(t, source.UpsertSession(ctx, sess)) + message := db.Message{ + SessionID: thread, Role: "assistant", Content: "retained page answer", + Timestamp: "2026-09-25T12:00:00Z", + } + require.NoError(t, source.InsertMessages(ctx, []db.Message{message})) + require.NoError(t, source.SetSessionDataVersion(ctx, thread, 130)) + projects := storage.PusherOptions{Projects: []string{"sample"}} + syncer, err := New(pgURL, schema, source, "machine", true, projects) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, syncer.Close()) }) + pushed, err := syncer.Push(ctx, true, nil) + require.NoError(t, err) + require.Zero(t, pushed.Errors) + store := &Store{pg: syncer.pg} + if curated { + require.NoError(t, store.RenameSession(ctx, thread, new("remote page title"))) + starred, err := store.StarSession(ctx, thread) + require.NoError(t, err) + require.True(t, starred) + _, err = store.PinMessage(ctx, thread, 0, new("remote note")) + require.NoError(t, err) + } - upgraded, err := New(pgURL, schema, rebuilt, "machine", true, projects) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, upgraded.Close()) }) - _, err = upgraded.Push(ctx, true, nil) - require.ErrorContains(t, err, `requires including project "other"`) + require.NoError(t, source.Close()) + raw, err := sql.Open("sqlite3", sourcePath) + require.NoError(t, err) + _, err = raw.ExecContext(ctx, "PRAGMA user_version=130") + require.NoError(t, err) + require.NoError(t, raw.Close()) + rebuilt := testDB(t) + require.NoError(t, rebuilt.CopyArchiveIdentityFrom(sourcePath)) + require.NoError(t, rebuilt.CopySyncStateFrom(sourcePath)) + sess.ID = page + sess.Project = "other" + sess.ParentSessionID = new(thread) + sess.RelationshipType = "continuation" + require.NoError(t, rebuilt.UpsertSession(ctx, sess)) + message.SessionID = page + require.NoError(t, rebuilt.InsertMessages(ctx, []db.Message{message})) + require.NoError(t, rebuilt.SetSessionDataVersion(ctx, page, db.CurrentDataVersion())) - row, err := store.GetSessionFull(ctx, thread) - require.NoError(t, err) - require.NotNil(t, row, "the push deleted the thread row") - require.NotNil(t, row.DisplayName) - assert.Equal(t, "remote page title", *row.DisplayName) - pins, err := store.ListPinnedMessages(ctx, thread, "") - require.NoError(t, err) - assert.Len(t, pins, 1) + upgraded, err := New(pgURL, schema, rebuilt, "machine", true, projects) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, upgraded.Close()) }) + _, err = upgraded.Push(ctx, true, nil) + row, getErr := store.GetSessionFull(ctx, thread) + require.NoError(t, getErr) + if !curated { + require.NoError(t, err) + assert.Nil(t, row, "the push kept a thread row with nothing to preserve") + return + } + require.ErrorContains(t, err, + `kept Codex thread `+thread+`: its pins and other curation move to its page in project "other"`) + require.NotNil(t, row, "the push deleted the thread row") + require.NotNil(t, row.DisplayName) + assert.Equal(t, "remote page title", *row.DisplayName) + pins, err := store.ListPinnedMessages(ctx, thread, "") + require.NoError(t, err) + assert.Len(t, pins, 1) + }) + } } diff --git a/internal/postgres/push.go b/internal/postgres/push.go index f01a3f4bb3..4312c8dc6e 100644 --- a/internal/postgres/push.go +++ b/internal/postgres/push.go @@ -2160,9 +2160,21 @@ func (s *Sync) reconcilePGMissingSessions( if err != nil { return fmt.Errorf("reading retained Codex page: %w", err) } - if page != nil { - pages = append(pages, *page) + if page == nil { + continue + } + if projectFailsFilter(page.Project, s.projects, s.excludeProjects) { + // An excluded page fails the push only when the thread has + // curation to move. Otherwise the thread row goes. + curated, err := s.codexThreadCurated(ctx, id) + if err != nil { + return err + } + if !curated { + continue + } } + pages = append(pages, *page) } if len(missing) == 0 { return nil @@ -4578,3 +4590,20 @@ func (s *Sync) syncCursorUsageEvents(ctx context.Context) error { } return nil } + +// codexThreadCurated reports whether a PostgreSQL thread row holds curation a +// user added remotely: pins, a star, a rename, or a trash change. +func (s *Sync) codexThreadCurated(ctx context.Context, threadID string) (bool, error) { + var curated bool + if err := s.pg.QueryRowContext(ctx, ` + SELECT EXISTS (SELECT 1 FROM pinned_messages WHERE session_id = $1) + OR EXISTS (SELECT 1 FROM starred_sessions WHERE session_id = $1) + OR EXISTS (SELECT 1 FROM sessions WHERE id = $1 AND ( + display_name IS DISTINCT FROM source_display_name + OR deleted_at IS DISTINCT FROM source_deleted_at))`, + threadID, + ).Scan(&curated); err != nil { + return false, fmt.Errorf("checking Codex thread curation: %w", err) + } + return curated, nil +} diff --git a/internal/sync/engine.go b/internal/sync/engine.go index 93e1683251..6436d714fe 100644 --- a/internal/sync/engine.go +++ b/internal/sync/engine.go @@ -4985,6 +4985,36 @@ func (e *Engine) RunExclusive(work func() error) error { return work() } +// RunExclusiveCtx is RunExclusive for a caller that must stop waiting for the +// lock when ctx ends, such as a push holding the daemon's push turn. It +// returns ctx.Err() without running work if ctx ends before the lock is free. +func (e *Engine) RunExclusiveCtx(ctx context.Context, work func() error) error { + if e.refuseWriteInForceParse("RunExclusiveCtx") { + return errors.New( + "RunExclusiveCtx refused on report-only parse-diff engine", + ) + } + if err := ctx.Err(); err != nil { + return err + } + locked := make(chan struct{}) + go func() { + e.syncMu.Lock() + close(locked) + }() + select { + case <-locked: + case <-ctx.Done(): + go func() { + <-locked + e.syncMu.Unlock() + }() + return ctx.Err() + } + defer e.syncMu.Unlock() + return work() +} + // ErrSyncInProgress reports that non-blocking foreground sync coordination // found another sync or maintenance pass holding the exclusive engine lock. var ErrSyncInProgress = errors.New("sync already in progress") diff --git a/internal/sync/signal_schedule_test.go b/internal/sync/signal_schedule_test.go index af1a54db65..51f24ae870 100644 --- a/internal/sync/signal_schedule_test.go +++ b/internal/sync/signal_schedule_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "sync" + "sync/atomic" "testing" "testing/synctest" "time" @@ -699,3 +700,49 @@ func TestSignalSchedulerRealTimerFlushes(t *testing.T) { "deferred recompute should flush via the real timer") }) } + +// Friction review before a push takes the engine lock. A canceled push must +// not wait out a sync that holds it, and must not run its work afterwards. +func TestRunExclusiveCtxStopsWaitingWhenContextEnds(t *testing.T) { + fx := newEngineFixture(t) + held := make(chan struct{}) + release := make(chan struct{}) + releaseLock := sync.OnceFunc(func() { close(release) }) + t.Cleanup(releaseLock) + exclusive := make(chan error, 1) + go func() { + exclusive <- fx.engine.RunExclusive(func() error { + close(held) + <-release + return nil + }) + }() + <-held + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var ran atomic.Bool + waited := make(chan error, 1) + go func() { + waited <- fx.engine.RunExclusiveCtx(ctx, func() error { + ran.Store(true) + return nil + }) + }() + require.Eventually(t, func() bool { + return testutil.GoroutineBlockedOnMutexIn("(*Engine).RunExclusiveCtx") + }, 10*time.Second, 5*time.Millisecond) + cancel() + select { + case err := <-waited: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + require.Fail(t, "the canceled caller kept waiting for the lock") + } + + releaseLock() + require.NoError(t, <-exclusive) + require.NoError(t, fx.engine.RunExclusiveCtx(t.Context(), func() error { return nil }), + "the abandoned wait must release the lock") + assert.False(t, ran.Load(), "the canceled work ran") +} From d9ea566757c151563982c1165b4a016d09dd1658 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:00:51 -0400 Subject: [PATCH 04/11] fix(push): let a canceled DuckDB push stop waiting for its cleanup After a DuckDB push succeeds, it deletes old pre-schema-v3 sync-state rows. That delete waited on the archive write lock without watching the push's context, so a long local write could keep a canceled push holding the daemon's push turn. The other archive writes on the push path already stop waiting on cancel. The cleanup is best effort, so a canceled push only skips it until the next one. The archive lock and the engine's sync lock each had a copy of the same wait-or-cancel code. That code has to hand the lock back after a wait gives up, which is easy to break, so it now lives in one place with its own test. Co-Authored-By: Claude Opus 5.5 --- internal/db/db.go | 4 +- internal/db/sessions.go | 23 ++------ internal/db/sessions_sync_marker_test.go | 7 ++- internal/lockctx/lockctx.go | 32 +++++++++++ internal/lockctx/lockctx_test.go | 67 ++++++++++++++++++++++++ internal/postgres/push_pgtest_test.go | 2 +- internal/sync/engine.go | 17 +----- internal/sync/signal_schedule_test.go | 2 +- 8 files changed, 116 insertions(+), 38 deletions(-) create mode 100644 internal/lockctx/lockctx.go create mode 100644 internal/lockctx/lockctx_test.go diff --git a/internal/db/db.go b/internal/db/db.go index 02505be2a4..78a124f3ae 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -5671,7 +5671,9 @@ func (db *DB) SetSyncStatesForDatabase( // the mirror's own sync_metadata table instead of local pg_sync_state). // prefix is escaped so LIKE metacharacters in it (%, _) match literally. func (db *DB) DeleteSyncStateByPrefix(ctx context.Context, prefix string) error { - db.mu.Lock() + if err := db.lockWriterCtx(ctx); err != nil { + return err + } defer db.mu.Unlock() escaped := escapeSyncStatePrefix(prefix) _, err := db.getWriter().Exec(ctx, diff --git a/internal/db/sessions.go b/internal/db/sessions.go index 766f1f0eb7..1a3bc2b952 100644 --- a/internal/db/sessions.go +++ b/internal/db/sessions.go @@ -18,6 +18,7 @@ import ( "strings" "time" + "go.kenn.io/agentsview/internal/lockctx" "go.kenn.io/agentsview/internal/parser" ) @@ -6826,26 +6827,12 @@ func (db *DB) MirrorPushCutoff(ctx context.Context) (string, error) { } } -// lockWriterCtx takes the write lock in the lock's normal queue, so a caller -// is not starved by back-to-back writes, and returns if ctx ends first. A -// request that gives up leaves a goroutine that releases the lock once it is -// granted. +// lockWriterCtx takes the write lock, or returns once ctx ends. func (db *DB) lockWriterCtx(ctx context.Context) error { - locked := make(chan struct{}) - go func() { - db.mu.Lock() - close(locked) - }() - select { - case <-locked: - return nil - case <-ctx.Done(): - go func() { - <-locked - db.mu.Unlock() - }() - return fmt.Errorf("stopped while waiting for an archive write to finish: %w", ctx.Err()) + if err := lockctx.Lock(ctx, &db.mu); err != nil { + return fmt.Errorf("stopped while waiting for an archive write to finish: %w", err) } + return nil } // WaitForWriter blocks while the writer is closed for a worker handoff. It diff --git a/internal/db/sessions_sync_marker_test.go b/internal/db/sessions_sync_marker_test.go index 467f18b81b..6959e38e2e 100644 --- a/internal/db/sessions_sync_marker_test.go +++ b/internal/db/sessions_sync_marker_test.go @@ -538,7 +538,7 @@ func TestMirrorPushCutoffCoversOpenWriteTransaction(t *testing.T) { // lock, so a push that ignores the open transaction lists first. var got listing received := false - for !received && !testutil.GoroutineBlockedOnMutexIn("(*DB).lockWriterCtx") { + for !received && !testutil.GoroutineBlockedOnMutexIn("lockctx.Lock") { select { case got = <-listed: received = true @@ -722,6 +722,9 @@ func TestPushStateWritesStopWaitingBehindALongWrite(t *testing.T) { "Update": func(ctx context.Context, d *DB) error { return d.Update(ctx, func(*sql.Tx) error { return nil }) }, + "DeleteSyncStateByPrefix": func(ctx context.Context, d *DB) error { + return d.DeleteSyncStateByPrefix(ctx, "duckdb_") + }, } { t.Run(name, func(t *testing.T) { database := testDB(t) @@ -743,7 +746,7 @@ func TestPushStateWritesStopWaitingBehindALongWrite(t *testing.T) { saved := make(chan error, 1) go func() { saved <- write(ctx, database) }() require.Eventually(t, func() bool { - return testutil.GoroutineBlockedOnMutexIn("(*DB).lockWriterCtx") + return testutil.GoroutineBlockedOnMutexIn("lockctx.Lock") }, 10*time.Second, 5*time.Millisecond) cancel() select { diff --git a/internal/lockctx/lockctx.go b/internal/lockctx/lockctx.go new file mode 100644 index 0000000000..90f03ce855 --- /dev/null +++ b/internal/lockctx/lockctx.go @@ -0,0 +1,32 @@ +// Package lockctx takes a mutex but stops waiting when a context ends. +package lockctx + +import ( + "context" + "sync" +) + +// Lock takes mu in the lock's normal queue, so the caller is not starved by +// back-to-back holders, and returns ctx.Err() if ctx ends first. On success +// the caller owns mu and must unlock it. A wait that gives up leaves a +// goroutine that unlocks mu once it is granted. +func Lock(ctx context.Context, mu sync.Locker) error { + if err := ctx.Err(); err != nil { + return err + } + locked := make(chan struct{}) + go func() { + mu.Lock() + close(locked) + }() + select { + case <-locked: + return nil + case <-ctx.Done(): + go func() { + <-locked + mu.Unlock() + }() + return ctx.Err() + } +} diff --git a/internal/lockctx/lockctx_test.go b/internal/lockctx/lockctx_test.go new file mode 100644 index 0000000000..789bb6623d --- /dev/null +++ b/internal/lockctx/lockctx_test.go @@ -0,0 +1,67 @@ +package lockctx + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/testutil" +) + +func TestLockTakesAFreeMutex(t *testing.T) { + var mu sync.Mutex + require.NoError(t, Lock(t.Context(), &mu)) + assert.False(t, mu.TryLock(), "Lock returned without holding the mutex") + mu.Unlock() +} + +func TestLockStopsWaitingAndReleasesTheMutexLater(t *testing.T) { + var mu sync.Mutex + mu.Lock() + release := sync.OnceFunc(mu.Unlock) + t.Cleanup(release) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + waited := make(chan error, 1) + go func() { waited <- Lock(ctx, &mu) }() + require.Eventually(t, func() bool { + return testutil.GoroutineBlockedOnMutexIn("lockctx.Lock") + }, 10*time.Second, 5*time.Millisecond) + cancel() + select { + case err := <-waited: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + require.Fail(t, "the canceled caller kept waiting for the mutex") + } + + release() + // Let the abandoned wait take the mutex first, so the next Lock checks + // that it gave the mutex back. + require.Eventually(t, func() bool { + return !testutil.GoroutineBlockedOnMutexIn("lockctx.Lock") + }, 10*time.Second, 5*time.Millisecond) + locked := make(chan error, 1) + go func() { locked <- Lock(t.Context(), &mu) }() + select { + case err := <-locked: + require.NoError(t, err) + mu.Unlock() + case <-time.After(5 * time.Second): + require.Fail(t, "the abandoned wait kept the mutex") + } +} + +func TestLockRefusesAnEndedContext(t *testing.T) { + var mu sync.Mutex + ctx, cancel := context.WithCancel(t.Context()) + cancel() + require.ErrorIs(t, Lock(ctx, &mu), context.Canceled) + assert.True(t, mu.TryLock(), "an ended context still took the mutex") + mu.Unlock() +} diff --git a/internal/postgres/push_pgtest_test.go b/internal/postgres/push_pgtest_test.go index 189f737de9..67a6e3434c 100644 --- a/internal/postgres/push_pgtest_test.go +++ b/internal/postgres/push_pgtest_test.go @@ -3774,7 +3774,7 @@ func TestFilteredPushReconcilesProjectMoveCommittedDuringPush(t *testing.T) { // finished without waiting. var pushErr error finished := false - for !finished && !testutil.GoroutineBlockedOnMutexIn("(*DB).lockWriterCtx") { + for !finished && !testutil.GoroutineBlockedOnMutexIn("lockctx.Lock") { select { case pushErr = <-pushed: finished = true diff --git a/internal/sync/engine.go b/internal/sync/engine.go index 6436d714fe..ee26e496ec 100644 --- a/internal/sync/engine.go +++ b/internal/sync/engine.go @@ -30,6 +30,7 @@ import ( "go.kenn.io/agentsview/internal/export" "go.kenn.io/agentsview/internal/friction" "go.kenn.io/agentsview/internal/ingest" + "go.kenn.io/agentsview/internal/lockctx" "go.kenn.io/agentsview/internal/parser" "go.kenn.io/agentsview/internal/pathutil" "go.kenn.io/agentsview/internal/secrets" @@ -4994,23 +4995,9 @@ func (e *Engine) RunExclusiveCtx(ctx context.Context, work func() error) error { "RunExclusiveCtx refused on report-only parse-diff engine", ) } - if err := ctx.Err(); err != nil { + if err := lockctx.Lock(ctx, &e.syncMu); err != nil { return err } - locked := make(chan struct{}) - go func() { - e.syncMu.Lock() - close(locked) - }() - select { - case <-locked: - case <-ctx.Done(): - go func() { - <-locked - e.syncMu.Unlock() - }() - return ctx.Err() - } defer e.syncMu.Unlock() return work() } diff --git a/internal/sync/signal_schedule_test.go b/internal/sync/signal_schedule_test.go index 51f24ae870..83a5f7c9b6 100644 --- a/internal/sync/signal_schedule_test.go +++ b/internal/sync/signal_schedule_test.go @@ -730,7 +730,7 @@ func TestRunExclusiveCtxStopsWaitingWhenContextEnds(t *testing.T) { }) }() require.Eventually(t, func() bool { - return testutil.GoroutineBlockedOnMutexIn("(*Engine).RunExclusiveCtx") + return testutil.GoroutineBlockedOnMutexIn("lockctx.Lock") }, 10*time.Second, 5*time.Millisecond) cancel() select { From d99bccaad1222abe28953a73698e773c28e2021f Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:00:54 -0400 Subject: [PATCH 05/11] fix(push): keep a Codex pin added while a filtered push runs A filtered full push decides whether to keep an old Codex thread whose page it excludes by checking the thread for pins, a star, a rename, or a trash change. That check ran before the delete transaction and took no lock. A pin that committed between the check and the delete was removed along with the thread. The push now locks the thread rows in the delete transaction and checks them there. Adding a pin locks the same row, so the check waits for it and sees it. The error a user sees in that case now says the push stopped and that the thread was kept. It also names the projects setting, because a daemon push has no flags. Co-Authored-By: Claude Opus 5.5 --- internal/postgres/codex_page_pins.go | 5 +- .../codex_page_retirement_pgtest_test.go | 39 +++++++++- internal/postgres/push.go | 73 +++++++++++++------ 3 files changed, 90 insertions(+), 27 deletions(-) diff --git a/internal/postgres/codex_page_pins.go b/internal/postgres/codex_page_pins.go index 2ff9aa0207..61698bf6b8 100644 --- a/internal/postgres/codex_page_pins.go +++ b/internal/postgres/codex_page_pins.go @@ -153,8 +153,9 @@ func (s *Sync) migrateCodexPages( } if projectFailsFilter(page.Project, s.projects, s.excludeProjects) { return fmt.Errorf( - "kept Codex thread %s: its pins and other curation move to its page in project %q, which this push excludes; "+ - "include that project with --projects, or remove it from --exclude-projects or the exclude_projects setting", + "push stopped: Codex thread %s has pins or other edits that must move to its page in project %q, "+ + "which this push excludes, so the thread was kept; add that project to --projects or the projects setting, "+ + "or remove it from --exclude-projects or the exclude_projects setting", old.id, page.Project) } var pageExists bool diff --git a/internal/postgres/codex_page_retirement_pgtest_test.go b/internal/postgres/codex_page_retirement_pgtest_test.go index 5f91f2eca6..2ee013f354 100644 --- a/internal/postgres/codex_page_retirement_pgtest_test.go +++ b/internal/postgres/codex_page_retirement_pgtest_test.go @@ -7,6 +7,7 @@ import ( "fmt" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -394,8 +395,9 @@ func TestCodexPageUpgradeIncrementalPushCopiesFriction(t *testing.T) { // holds the only copy of its pins, star, and title. A thread with no curation // has nothing to lose, so the push deletes it and succeeds. func TestFilteredFullPushKeepsCuratedCodexThreadWhosePageIsExcluded(t *testing.T) { - for _, curated := range []bool{true, false} { - t.Run(fmt.Sprintf("curated=%t", curated), func(t *testing.T) { + for _, mode := range []string{"curated", "uncurated", "pinned during push"} { + t.Run(mode, func(t *testing.T) { + curated := mode == "curated" const thread = "codex:11111111-1111-4111-8111-111111111111" const page = thread + "_22222222-2222-4222-8222-222222222222" const schema = "agentsview_codex_page_filtered_cleanup_test" @@ -455,6 +457,37 @@ func TestFilteredFullPushKeepsCuratedCodexThreadWhosePageIsExcluded(t *testing.T upgraded, err := New(pgURL, schema, rebuilt, "machine", true, projects) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, upgraded.Close()) }) + if mode == "pinned during push" { + // A pin that commits while the push runs must reach the + // curation check, or the push deletes it with the thread. + pin, err := syncer.pg.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = pin.Rollback() }) + require.NoError(t, lockPinnedMessagesSession(ctx, pin, thread)) + _, err = pinMessageTx(ctx, pin, thread, 0, new("remote note")) + require.NoError(t, err) + pushed := make(chan error, 1) + go func() { + _, err := upgraded.Push(ctx, true, nil) + pushed <- err + }() + require.Eventually(t, func() bool { + var waiting bool + require.NoError(t, syncer.pg.QueryRowContext(ctx, ` + SELECT EXISTS (SELECT 1 FROM pg_stat_activity + WHERE datname = current_database() AND pid <> pg_backend_pid() + AND wait_event_type = 'Lock')`).Scan(&waiting)) + return waiting + }, 30*time.Second, 10*time.Millisecond, "the push never waited for the pin's row lock") + require.NoError(t, pin.Commit()) + err = <-pushed + require.ErrorContains(t, err, + `push stopped: Codex thread `+thread+` has pins or other edits that must move to its page in project "other"`) + pins, err := store.ListPinnedMessages(ctx, thread, "") + require.NoError(t, err) + assert.Len(t, pins, 1, "the push deleted a pin added while it ran") + return + } _, err = upgraded.Push(ctx, true, nil) row, getErr := store.GetSessionFull(ctx, thread) require.NoError(t, getErr) @@ -464,7 +497,7 @@ func TestFilteredFullPushKeepsCuratedCodexThreadWhosePageIsExcluded(t *testing.T return } require.ErrorContains(t, err, - `kept Codex thread `+thread+`: its pins and other curation move to its page in project "other"`) + `push stopped: Codex thread `+thread+` has pins or other edits that must move to its page in project "other"`) require.NotNil(t, row, "the push deleted the thread row") require.NotNil(t, row.DisplayName) assert.Equal(t, "remote page title", *row.DisplayName) diff --git a/internal/postgres/push.go b/internal/postgres/push.go index 4312c8dc6e..0f5866b778 100644 --- a/internal/postgres/push.go +++ b/internal/postgres/push.go @@ -2146,6 +2146,9 @@ func (s *Sync) reconcilePGMissingSessions( } var missing []string var pages []db.Session + // Threads whose page this push excludes, checked for curation under + // a row lock in the delete transaction below. + excluded := make(map[string]db.Session) for _, id := range chunk { if _, exists := present[id]; exists { continue @@ -2164,15 +2167,8 @@ func (s *Sync) reconcilePGMissingSessions( continue } if projectFailsFilter(page.Project, s.projects, s.excludeProjects) { - // An excluded page fails the push only when the thread has - // curation to move. Otherwise the thread row goes. - curated, err := s.codexThreadCurated(ctx, id) - if err != nil { - return err - } - if !curated { - continue - } + excluded[id] = *page + continue } pages = append(pages, *page) } @@ -2184,6 +2180,16 @@ func (s *Sync) reconcilePGMissingSessions( return err } defer func() { _ = tx.Rollback() }() + // An excluded page fails the push only when the thread has curation + // to move. Otherwise the thread row goes. The row lock makes a pin, + // star, or rename that commits during the push wait for this check. + curated, err := lockCuratedCodexThreads(ctx, tx, mapKeys(excluded)) + if err != nil { + return err + } + for _, id := range curated { + pages = append(pages, excluded[id]) + } // A filtered push skips a page outside its projects, so the batch push // never moved the old thread's pins, stars, and title onto it. Move // them now, or fail when the page is excluded, before the delete @@ -4591,19 +4597,42 @@ func (s *Sync) syncCursorUsageEvents(ctx context.Context) error { return nil } -// codexThreadCurated reports whether a PostgreSQL thread row holds curation a -// user added remotely: pins, a star, a rename, or a trash change. -func (s *Sync) codexThreadCurated(ctx context.Context, threadID string) (bool, error) { - var curated bool - if err := s.pg.QueryRowContext(ctx, ` - SELECT EXISTS (SELECT 1 FROM pinned_messages WHERE session_id = $1) - OR EXISTS (SELECT 1 FROM starred_sessions WHERE session_id = $1) - OR EXISTS (SELECT 1 FROM sessions WHERE id = $1 AND ( - display_name IS DISTINCT FROM source_display_name - OR deleted_at IS DISTINCT FROM source_deleted_at))`, - threadID, - ).Scan(&curated); err != nil { - return false, fmt.Errorf("checking Codex thread curation: %w", err) +// lockCuratedCodexThreads locks the PostgreSQL thread rows with ids and +// returns those holding curation a user added remotely: pins, a star, a +// rename, or a trash change. +func lockCuratedCodexThreads(ctx context.Context, tx *sql.Tx, ids []string) ([]string, error) { + if len(ids) == 0 { + return nil, nil + } + if _, err := tx.ExecContext(ctx, + `SELECT 1 FROM sessions WHERE id = ANY($1) ORDER BY id FOR UPDATE`, ids, + ); err != nil { + return nil, fmt.Errorf("locking Codex threads: %w", err) + } + rows, err := tx.QueryContext(ctx, ` + SELECT s.id FROM sessions s + WHERE s.id = ANY($1) AND ( + EXISTS (SELECT 1 FROM pinned_messages p WHERE p.session_id = s.id) + OR EXISTS (SELECT 1 FROM starred_sessions st WHERE st.session_id = s.id) + OR s.display_name IS DISTINCT FROM s.source_display_name + OR s.deleted_at IS DISTINCT FROM s.source_deleted_at) + ORDER BY s.id`, + ids, + ) + if err != nil { + return nil, fmt.Errorf("checking Codex thread curation: %w", err) + } + defer rows.Close() + var curated []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("checking Codex thread curation: %w", err) + } + curated = append(curated, id) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("checking Codex thread curation: %w", err) } return curated, nil } From 861f07de76eb1cb90022fd1b0d71e69aee03e039 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:58:15 -0400 Subject: [PATCH 06/11] fix(push): let a canceled push stop waiting during its pricing refresh Each push refreshes model pricing before it copies anything. The refresh got the push's context, but its steps switched to a background context, and the pricing writes waited on the archive write lock without checking any context. Local sync now keeps writing while a push runs, so a long sync write could keep a canceled push stuck in the refresh and holding the daemon's push turn. The refresh now passes the push's context down, and the pricing writes stop waiting when it ends. The exported Ensure, SeedFallback, and RefreshIfStale keep their signatures for the startup and usage callers. A canceled refresh puts back the previous refresh time so the next push can try again. That write can also sit behind a long sync write, so it now finishes in the background and holds the refresh gate until it is done. The canceled push returns at once, and the next refresh in this process waits for the restore. A failed restore is now logged instead of returned; the push has already failed with its cancellation. The cutoff cancel test now waits until the cutoff call is blocked on the lock before it cancels. Before, a cancel that landed first let the call return early, so the test could pass with a plain blocking lock. Co-Authored-By: Claude Opus 5.5 --- internal/db/genai_pricing.go | 4 +- internal/db/pricing.go | 8 ++- internal/db/sessions_sync_marker_test.go | 3 + internal/pricingrefresh/refresh.go | 89 +++++++++++++++--------- internal/pricingrefresh/refresh_test.go | 66 +++++++++++++++++- 5 files changed, 130 insertions(+), 40 deletions(-) diff --git a/internal/db/genai_pricing.go b/internal/db/genai_pricing.go index 80e92637b3..4b00a56c3c 100644 --- a/internal/db/genai_pricing.go +++ b/internal/db/genai_pricing.go @@ -122,7 +122,9 @@ func (db *DB) writeGenAIPricing( OR genai_pricing.source_ref IS NOT excluded.source_ref OR genai_pricing.data_json IS NOT excluded.data_json)` } - db.mu.Lock() + if err := db.lockWriterCtx(ctx); err != nil { + return err + } defer db.mu.Unlock() _, err := db.getWriter().ExecContext(ctx, ` INSERT INTO genai_pricing diff --git a/internal/db/pricing.go b/internal/db/pricing.go index 2be02215bc..20436fad3e 100644 --- a/internal/db/pricing.go +++ b/internal/db/pricing.go @@ -237,7 +237,9 @@ func (db *DB) ReconcileModelPricingContext( return nil } - db.mu.Lock() + if err := db.lockWriterCtx(ctx); err != nil { + return err + } defer db.mu.Unlock() existing, err := db.listModelPricing(ctx) @@ -403,7 +405,9 @@ func (db *DB) DeleteModelPricingContext( return nil } - db.mu.Lock() + if err := db.lockWriterCtx(ctx); err != nil { + return err + } defer db.mu.Unlock() placeholders := make([]string, len(patterns)) diff --git a/internal/db/sessions_sync_marker_test.go b/internal/db/sessions_sync_marker_test.go index 6959e38e2e..8c23687501 100644 --- a/internal/db/sessions_sync_marker_test.go +++ b/internal/db/sessions_sync_marker_test.go @@ -693,6 +693,9 @@ func TestMirrorPushCutoffStopsWaitingBehindALongWrite(t *testing.T) { _, err := database.MirrorPushCutoff(ctx) cutoff <- err }() + require.Eventually(t, func() bool { + return testutil.GoroutineBlockedOnMutexIn("lockctx.Lock") + }, 10*time.Second, 5*time.Millisecond) cancel() select { case err := <-cutoff: diff --git a/internal/pricingrefresh/refresh.go b/internal/pricingrefresh/refresh.go index e789abd31f..99d3d7f5bb 100644 --- a/internal/pricingrefresh/refresh.go +++ b/internal/pricingrefresh/refresh.go @@ -4,6 +4,7 @@ package pricingrefresh import ( "context" "fmt" + "log" "slices" "strings" "sync" @@ -133,7 +134,17 @@ func RefreshIfStale( cooldown time.Duration, now time.Time, ) (bool, error) { - stored, err := database.GetPricingMeta(refreshAttemptMetaKey) + return refreshIfStale(context.Background(), database, fetch, cooldown, now) +} + +func refreshIfStale( + ctx context.Context, + database *db.DB, + fetch func() (pricing.Catalog, error), + cooldown time.Duration, + now time.Time, +) (bool, error) { + stored, err := database.GetPricingMetaContext(ctx, refreshAttemptMetaKey) if err != nil { return false, fmt.Errorf("reading pricing refresh meta: %w", err) } @@ -143,16 +154,17 @@ func RefreshIfStale( return false, nil } } - return refreshAt(database, fetch, now) + return refreshAt(ctx, database, fetch, now) } func refreshAt( + ctx context.Context, database *db.DB, fetch func() (pricing.Catalog, error), now time.Time, ) (bool, error) { - if err := database.SetPricingMeta( - refreshAttemptMetaKey, now.UTC().Format(time.RFC3339), + if err := database.SetPricingMetaContext( + ctx, refreshAttemptMetaKey, now.UTC().Format(time.RFC3339), ); err != nil { return false, fmt.Errorf( "recording pricing refresh attempt: %w", err, @@ -162,7 +174,7 @@ func refreshAt( if fetchErr != nil && catalog.GenAI == nil && len(catalog.LiteLLM) == 0 { return false, fetchErr } - if err := storeCatalog(database, catalog); err != nil { + if err := storeCatalog(ctx, database, catalog); err != nil { return false, err } return true, fetchErr @@ -174,17 +186,27 @@ func Ensure( offline bool, fetch func() (pricing.Catalog, error), now time.Time, +) (bool, error) { + return ensure(context.Background(), database, offline, fetch, now) +} + +func ensure( + ctx context.Context, + database *db.DB, + offline bool, + fetch func() (pricing.Catalog, error), + now time.Time, ) (bool, error) { if offline { - if err := seedEmbeddedGenAI(context.Background(), database); err != nil { + if err := seedEmbeddedGenAI(ctx, database); err != nil { return false, err } - return false, storeFallback(database) + return false, storeFallbackContext(ctx, database) } - if err := SeedFallback(database); err != nil { + if err := SeedFallbackContext(ctx, database); err != nil { return false, err } - return RefreshIfStale(database, fetch, RefreshCooldown, now) + return refreshIfStale(ctx, database, fetch, RefreshCooldown, now) } // EnsureCurrent applies the standard online pricing lifecycle. @@ -243,36 +265,39 @@ func runCurrent( return ctx.Err() } } - defer func() { + releaseGate := func() { gate.slot <- struct{}{} releaseRefreshGateReference(database, gate) - }() + } - previousAttempt, err := database.GetPricingMeta(refreshAttemptMetaKey) + previousAttempt, err := database.GetPricingMetaContext(ctx, refreshAttemptMetaKey) if err != nil { + releaseGate() return fmt.Errorf("reading pricing refresh meta: %w", err) } fetchCurrent := func() (pricing.Catalog, error) { return fetch(ctx) } if force { - if err = SeedFallback(database); err == nil { - _, err = refreshAt(database, fetchCurrent, now) + if err = SeedFallbackContext(ctx, database); err == nil { + _, err = refreshAt(ctx, database, fetchCurrent, now) } } else { - _, err = Ensure(database, false, fetchCurrent, now) + _, err = ensure(ctx, database, false, fetchCurrent, now) } if err == nil || ctx.Err() == nil { + releaseGate() return err } - if restoreErr := database.SetPricingMeta( - refreshAttemptMetaKey, previousAttempt, - ); restoreErr != nil { - return fmt.Errorf( - "restoring pricing refresh attempt after cancellation: %w: %w", - restoreErr, err, - ) - } + // Put back the previous attempt time so the next refresh can retry. That + // write can wait behind a long archive write, and a canceled push must + // not, so it finishes in the background and holds the gate until then. + go func() { + defer releaseGate() + if err := database.SetPricingMeta(refreshAttemptMetaKey, previousAttempt); err != nil { + log.Printf("warning: restoring pricing refresh attempt after cancellation: %v", err) + } + }() return err } @@ -288,10 +313,10 @@ func runCurrent( // refresh recomputes ownership from the stored table and converges: // retiring an already-deleted row is a no-op and a re-listed pattern is // re-adopted. -func storeCatalog(database *db.DB, catalog pricing.Catalog) error { +func storeCatalog(ctx context.Context, database *db.DB, catalog pricing.Catalog) error { if catalog.GenAI != nil { if err := database.UpsertGenAIPricing( - context.Background(), db.GenAIPricingDocument{ + ctx, db.GenAIPricingDocument{ Version: catalog.GenAI.Version, SourceRef: catalog.GenAI.SourceRef, Source: db.GenAIPricingSourceFetched, Data: catalog.GenAI.RawJSON(), @@ -303,7 +328,7 @@ func storeCatalog(database *db.DB, catalog pricing.Catalog) error { if len(catalog.LiteLLM) == 0 && len(catalog.OpenRouter) == 0 { return nil } - value, err := database.GetPricingMeta(pricing.OpenRouterModelsMetaKey) + value, err := database.GetPricingMetaContext(ctx, pricing.OpenRouterModelsMetaKey) if err != nil { return err } @@ -311,7 +336,7 @@ func storeCatalog(database *db.DB, catalog pricing.Catalog) error { if err != nil { return err } - existing, err := database.ListModelPricing(context.Background()) + existing, err := database.ListModelPricing(ctx) if err != nil { return fmt.Errorf("listing stored pricing: %w", err) } @@ -322,8 +347,8 @@ func storeCatalog(database *db.DB, catalog pricing.Catalog) error { } } prices, owned, retired := catalog.Reconcile(stored, previous) - return database.ReconcileModelPricing( - dbModelPricing(prices), retired, + return database.ReconcileModelPricingContext( + ctx, dbModelPricing(prices), retired, db.PricingMeta{ Key: pricing.OpenRouterModelsMetaKey, Value: pricing.EncodeOpenRouterModels(owned), @@ -339,17 +364,13 @@ func seedEmbeddedGenAI(ctx context.Context, database *db.DB) error { }) } -// storeFallback writes the embedded catalog over the stored table. Like +// storeFallbackContext writes the embedded catalog over the stored table. Like // any non-OpenRouter source it outranks OpenRouter, so OpenRouter-owned // rows it covers under another spelling are retired and rows it lists // exactly pass to its ownership, in the same transaction, so a reseed // never leaves an OpenRouter row beside or under an embedded one while // the sentinel still claims it. Databases that never stored the sentinel // keep none. -func storeFallback(database *db.DB) error { - return storeFallbackContext(context.Background(), database) -} - func storeFallbackContext(ctx context.Context, database *db.DB) error { value, err := database.GetPricingMetaContext( ctx, pricing.OpenRouterModelsMetaKey, diff --git a/internal/pricingrefresh/refresh_test.go b/internal/pricingrefresh/refresh_test.go index 7118bc045b..32ba6e8566 100644 --- a/internal/pricingrefresh/refresh_test.go +++ b/internal/pricingrefresh/refresh_test.go @@ -2,9 +2,11 @@ package pricingrefresh import ( "context" + "database/sql" "errors" "path/filepath" "strings" + "sync" "sync/atomic" "testing" "testing/synctest" @@ -16,6 +18,7 @@ import ( "go.kenn.io/agentsview/internal/dbtest" "go.kenn.io/agentsview/internal/money" "go.kenn.io/agentsview/internal/pricing" + "go.kenn.io/agentsview/internal/testutil" ) const refreshAttemptMetaKeyForTest = "_litellm_last_attempt" @@ -302,7 +305,7 @@ func TestStoreCatalogRetiresShadowedOpenRouterRows(t *testing.T) { InputPerMTok: money.MustParseDollars("8"), }, } - require.NoError(t, storeCatalog(database, pricing.Catalog{ + require.NoError(t, storeCatalog(t.Context(), database, pricing.Catalog{ OpenRouter: openrouter, })) meta, err := database.GetPricingMeta(pricing.OpenRouterModelsMetaKey) @@ -314,7 +317,7 @@ func TestStoreCatalogRetiresShadowedOpenRouterRows(t *testing.T) { assert.Nil(t, shadowed) // LiteLLM now lists the model under its own spelling. - require.NoError(t, storeCatalog(database, pricing.Catalog{ + require.NoError(t, storeCatalog(t.Context(), database, pricing.Catalog{ LiteLLM: []pricing.ModelPricing{{ ModelPattern: "minimax/MiniMax-M3", InputPerMTok: money.MustParseDollars("2"), @@ -347,7 +350,7 @@ func TestStoreFallbackReconcilesOpenRouterOwnership(t *testing.T) { } } require.NotEmpty(t, spelled, "need a fallback pattern with lowercase") - require.NoError(t, storeCatalog(database, pricing.Catalog{ + require.NoError(t, storeCatalog(t.Context(), database, pricing.Catalog{ OpenRouter: []pricing.ModelPricing{ {ModelPattern: exact, InputPerMTok: money.MustParseDollars("99")}, {ModelPattern: spelled, InputPerMTok: money.MustParseDollars("98")}, @@ -393,6 +396,7 @@ func TestEnsureCurrentCancellationAllowsImmediateRetry(t *testing.T) { }, now) require.ErrorIs(t, err, context.Canceled) + waitForRefreshGate(database) assertPricingAttemptMeta(t, database, previous) retryCalls := 0 err = ensureCurrent(t.Context(), database, func( @@ -406,6 +410,53 @@ func TestEnsureCurrentCancellationAllowsImmediateRetry(t *testing.T) { assert.Equal(t, 1, retryCalls) } +// A push refreshes pricing first. Canceling the push must end the refresh +// even while a long archive write holds the lock its pricing writes need. +func TestEnsureCurrentStopsWaitingBehindALongWrite(t *testing.T) { + database := testDB(t) + now := pricingTestNow() + previous := seedPricingAttempt(t, database, now, 2*time.Hour) + inWrite := make(chan struct{}) + release := make(chan struct{}) + endWrite := sync.OnceFunc(func() { close(release) }) + t.Cleanup(endWrite) + writeDone := make(chan error, 1) + go func() { + writeDone <- database.Update(t.Context(), func(*sql.Tx) error { + close(inWrite) + <-release + return nil + }) + }() + <-inWrite + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + refreshed := make(chan error, 1) + go func() { + refreshed <- ensureCurrent(ctx, database, func( + context.Context, + ) (pricing.Catalog, error) { + return pricing.Catalog{}, nil + }, now) + }() + require.Eventually(t, func() bool { + return testutil.GoroutineBlockedOnMutexIn("lockctx.Lock") + }, 10*time.Second, 5*time.Millisecond) + cancel() + select { + case err := <-refreshed: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + require.Fail(t, "the canceled refresh kept waiting for the archive write") + } + + endWrite() + require.NoError(t, <-writeDone) + waitForRefreshGate(database) + assertPricingAttemptMeta(t, database, previous) +} + func TestRefreshCurrentFetchesDespiteRecentAttempt(t *testing.T) { database := testDB(t) now := pricingTestNow() @@ -506,6 +557,15 @@ func testDB(t *testing.T) *db.DB { ) } +// waitForRefreshGate returns once no refresh holds the gate, including the +// background restore after a canceled refresh. +func waitForRefreshGate(database *db.DB) { + gate := retainRefreshGate(database) + <-gate.slot + gate.slot <- struct{}{} + releaseRefreshGateReference(database, gate) +} + func pricingTestNow() time.Time { return time.Date(2026, 5, 25, 12, 0, 0, 0, time.UTC) } From d9ecc889fa8006c7fa3e62d22e01b2e5b4daf8d8 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:37:13 -0400 Subject: [PATCH 07/11] fix(pricing): restore the refresh time before a canceled refresh returns The previous commit moved the restore of the last refresh time into a background goroutine, so a canceled push would not wait for it. Nothing waited for that goroutine. A CLI push that exits, or a daemon that closes the archive on shutdown, could drop the write, and the next push within the hour-long cooldown would then skip the pricing refresh even though no catalog was fetched. The restore runs before return again. It is skipped when the canceled refresh never recorded an attempt, which is the case when the cancel lands while the refresh is still waiting for the archive lock. Only a cancel that lands after the attempt was recorded, during the fetch or the catalog write, can still wait behind a long archive write to put the old time back. Co-Authored-By: Claude Opus 5.5 --- internal/pricingrefresh/refresh.go | 31 ++++++++++++++----------- internal/pricingrefresh/refresh_test.go | 11 --------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/internal/pricingrefresh/refresh.go b/internal/pricingrefresh/refresh.go index 99d3d7f5bb..f4589c77af 100644 --- a/internal/pricingrefresh/refresh.go +++ b/internal/pricingrefresh/refresh.go @@ -4,7 +4,6 @@ package pricingrefresh import ( "context" "fmt" - "log" "slices" "strings" "sync" @@ -265,14 +264,13 @@ func runCurrent( return ctx.Err() } } - releaseGate := func() { + defer func() { gate.slot <- struct{}{} releaseRefreshGateReference(database, gate) - } + }() previousAttempt, err := database.GetPricingMetaContext(ctx, refreshAttemptMetaKey) if err != nil { - releaseGate() return fmt.Errorf("reading pricing refresh meta: %w", err) } fetchCurrent := func() (pricing.Catalog, error) { @@ -286,18 +284,23 @@ func runCurrent( _, err = ensure(ctx, database, false, fetchCurrent, now) } if err == nil || ctx.Err() == nil { - releaseGate() return err } - // Put back the previous attempt time so the next refresh can retry. That - // write can wait behind a long archive write, and a canceled push must - // not, so it finishes in the background and holds the gate until then. - go func() { - defer releaseGate() - if err := database.SetPricingMeta(refreshAttemptMetaKey, previousAttempt); err != nil { - log.Printf("warning: restoring pricing refresh attempt after cancellation: %v", err) - } - }() + // Put back the previous attempt time so the next refresh can retry. Skip + // the write when this refresh never recorded an attempt: it can wait + // behind a long archive write, and a canceled push should not. + current, readErr := database.GetPricingMeta(refreshAttemptMetaKey) + if readErr == nil && current == previousAttempt { + return err + } + if restoreErr := database.SetPricingMeta( + refreshAttemptMetaKey, previousAttempt, + ); restoreErr != nil { + return fmt.Errorf( + "restoring pricing refresh attempt after cancellation: %w: %w", + restoreErr, err, + ) + } return err } diff --git a/internal/pricingrefresh/refresh_test.go b/internal/pricingrefresh/refresh_test.go index 32ba6e8566..076a36b050 100644 --- a/internal/pricingrefresh/refresh_test.go +++ b/internal/pricingrefresh/refresh_test.go @@ -396,7 +396,6 @@ func TestEnsureCurrentCancellationAllowsImmediateRetry(t *testing.T) { }, now) require.ErrorIs(t, err, context.Canceled) - waitForRefreshGate(database) assertPricingAttemptMeta(t, database, previous) retryCalls := 0 err = ensureCurrent(t.Context(), database, func( @@ -453,7 +452,6 @@ func TestEnsureCurrentStopsWaitingBehindALongWrite(t *testing.T) { endWrite() require.NoError(t, <-writeDone) - waitForRefreshGate(database) assertPricingAttemptMeta(t, database, previous) } @@ -557,15 +555,6 @@ func testDB(t *testing.T) *db.DB { ) } -// waitForRefreshGate returns once no refresh holds the gate, including the -// background restore after a canceled refresh. -func waitForRefreshGate(database *db.DB) { - gate := retainRefreshGate(database) - <-gate.slot - gate.slot <- struct{}{} - releaseRefreshGateReference(database, gate) -} - func pricingTestNow() time.Time { return time.Date(2026, 5, 25, 12, 0, 0, 0, time.UTC) } From e055588380eee147bda3125b3ff943d2a34a9400 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:48:17 -0400 Subject: [PATCH 08/11] fix(push): keep PostgreSQL pins that a concurrent sync or restore would drop Local sync now runs while a push copies transcripts, and the copy reads each session 1,000 messages at a time. If sync rewrote the transcript between two reads and a pinned message moved into a page the push had already read, the copy missed it, and the pin restore deleted the pin. A later push repaired the transcript but could not bring the pin back. The push now reads the session's transcript revision before and after the copy, and copies again when it changed, up to three times. Every local message write bumps that revision in the same transaction, so the check sees any rewrite. Full-push cleanup also deleted a legacy Codex thread that had been trashed and restored remotely. The restore stamps a current data version, so page migration no longer recognizes the row as legacy and leaves its pins, star, and title on it, and the delete then cascaded them away. Migration deletes a thread once it has moved its curation, so cleanup now keeps any page-backed thread that still holds curation after migration. That leaves the old row in place, which is what happened before this PR added the cleanup. Not changed: a push canceled after its pricing refresh recorded an attempt still waits for the archive writer to put the old time back. roborev-ci reported this on d9ecc88; the wait lasts at most one sync write, and skipping the restore would only delay the next refresh. Co-Authored-By: Claude Opus 5.5 --- .../codex_page_retirement_pgtest_test.go | 68 +++++++++++++++ internal/postgres/push.go | 86 ++++++++++++++++--- .../postgres/push_transcript_pgtest_test.go | 69 +++++++++++++++ internal/postgres/sync.go | 2 + 4 files changed, 214 insertions(+), 11 deletions(-) create mode 100644 internal/postgres/push_transcript_pgtest_test.go diff --git a/internal/postgres/codex_page_retirement_pgtest_test.go b/internal/postgres/codex_page_retirement_pgtest_test.go index 2ee013f354..b2b09536a3 100644 --- a/internal/postgres/codex_page_retirement_pgtest_test.go +++ b/internal/postgres/codex_page_retirement_pgtest_test.go @@ -507,3 +507,71 @@ func TestFilteredFullPushKeepsCuratedCodexThreadWhosePageIsExcluded(t *testing.T }) } } + +// Restoring a legacy Codex thread remotely stamps it with a current data +// version, so page migration no longer recognizes it. A full push after an +// archive rebuild must keep the thread and its curation rather than delete it. +func TestFullPushKeepsRemotelyRestoredCodexThread(t *testing.T) { + const thread = "codex:11111111-1111-4111-8111-111111111111" + const page = thread + "_22222222-2222-4222-8222-222222222222" + const schema = "agentsview_codex_page_restored_cleanup_test" + ctx := t.Context() + pgURL := testPGURL(t) + cleanNamedPGSchema(t, pgURL, schema) + t.Cleanup(func() { cleanNamedPGSchema(t, pgURL, schema) }) + source := testDB(t) + sourcePath := source.Path() + pagePath := filepath.Join(t.TempDir(), "rollout-2026-09-25T12-00-00-"+page[len("codex:"):]+".jsonl") + sess := db.Session{ + ID: thread, Agent: "codex", Project: "sample", Machine: "machine", + FilePath: &pagePath, MessageCount: 1, CreatedAt: "2026-09-25T12:00:00Z", + } + require.NoError(t, source.UpsertSession(ctx, sess)) + message := db.Message{ + SessionID: thread, Role: "assistant", Content: "retained page answer", + Timestamp: "2026-09-25T12:00:00Z", + } + require.NoError(t, source.InsertMessages(ctx, []db.Message{message})) + require.NoError(t, source.SetSessionDataVersion(ctx, thread, 130)) + syncer, err := New(pgURL, schema, source, "machine", true, storage.PusherOptions{}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, syncer.Close()) }) + pushed, err := syncer.Push(ctx, true, nil) + require.NoError(t, err) + require.Zero(t, pushed.Errors) + store := &Store{pg: syncer.pg} + _, err = store.PinMessage(ctx, thread, 0, new("remote note")) + require.NoError(t, err) + require.NoError(t, store.SoftDeleteSession(ctx, thread)) + _, err = store.RestoreSession(ctx, thread) + require.NoError(t, err) + + require.NoError(t, source.Close()) + raw, err := sql.Open("sqlite3", sourcePath) + require.NoError(t, err) + _, err = raw.ExecContext(ctx, "PRAGMA user_version=130") + require.NoError(t, err) + require.NoError(t, raw.Close()) + rebuilt := testDB(t) + require.NoError(t, rebuilt.CopyArchiveIdentityFrom(sourcePath)) + require.NoError(t, rebuilt.CopySyncStateFrom(sourcePath)) + sess.ID = page + sess.ParentSessionID = new(thread) + sess.RelationshipType = "continuation" + require.NoError(t, rebuilt.UpsertSession(ctx, sess)) + message.SessionID = page + require.NoError(t, rebuilt.InsertMessages(ctx, []db.Message{message})) + require.NoError(t, rebuilt.SetSessionDataVersion(ctx, page, db.CurrentDataVersion())) + + upgraded, err := New(pgURL, schema, rebuilt, "machine", true, storage.PusherOptions{}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, upgraded.Close()) }) + _, err = upgraded.Push(ctx, true, nil) + require.NoError(t, err) + + pins, err := store.ListPinnedMessages(ctx, thread, "") + require.NoError(t, err) + pagePins, err := store.ListPinnedMessages(ctx, page, "") + require.NoError(t, err) + assert.Len(t, append(pins, pagePins...), 1, "the push deleted the restored thread's pin") +} diff --git a/internal/postgres/push.go b/internal/postgres/push.go index 0f5866b778..fc5577b04e 100644 --- a/internal/postgres/push.go +++ b/internal/postgres/push.go @@ -2149,6 +2149,8 @@ func (s *Sync) reconcilePGMissingSessions( // Threads whose page this push excludes, checked for curation under // a row lock in the delete transaction below. excluded := make(map[string]db.Session) + // Threads with a retained page, whose curation migration should move. + var paged []string for _, id := range chunk { if _, exists := present[id]; exists { continue @@ -2166,6 +2168,7 @@ func (s *Sync) reconcilePGMissingSessions( if page == nil { continue } + paged = append(paged, id) if projectFailsFilter(page.Project, s.projects, s.excludeProjects) { excluded[id] = *page continue @@ -2197,6 +2200,17 @@ func (s *Sync) reconcilePGMissingSessions( if err := s.migrateCodexPages(ctx, tx, pages, ownerMarker, legacyMarkerMachines); err != nil { return err } + // Migration retires a thread once its curation is on the page. A + // thread it skipped, such as a legacy thread restored remotely, still + // holds that curation, so keep the thread rather than delete it. + kept, err := lockCuratedCodexThreads(ctx, tx, paged) + if err != nil { + return err + } + if len(kept) > 0 { + log.Printf("postgres push: kept %d Codex thread(s) whose curation did not move to their page", len(kept)) + missing = slices.DeleteFunc(missing, func(id string) bool { return slices.Contains(kept, id) }) + } // Recheck ownership and scope, and clear vectors only for deleted rows. rows, err := tx.QueryContext(ctx, "DELETE FROM sessions WHERE "+where+fmt.Sprintf(" AND id = ANY($%d) RETURNING id", len(args)+1), @@ -3296,6 +3310,64 @@ func (s *Sync) pushMessages( if err != nil { return 0, err } + if err := s.replaceUsageEvents(ctx, tx, sessionID); err != nil { + return 0, err + } + // Local sync can rewrite this transcript while the copy reads it page by + // page. A pinned message that moves across a page boundary would be + // missed, and restoring the pins would then drop its pin, so copy again + // whenever the transcript changed during the copy. + var count int + for attempt := 1; ; attempt++ { + before, err := s.localTranscriptRevision(ctx, sessionID) + if err != nil { + return 0, err + } + count, err = s.copySessionMessages(ctx, tx, sessionID) + if err != nil { + return count, err + } + after, err := s.localTranscriptRevision(ctx, sessionID) + if err != nil { + return count, err + } + if before == after { + break + } + if attempt == maxTranscriptCopyAttempts { + return count, fmt.Errorf( + "pushMessages %s: the transcript changed during %d copies; push again", + sessionID, attempt, + ) + } + } + + if err := restorePinnedMessages( + ctx, tx, sessionID, savedPins, + ); err != nil { + return count, err + } + + return count, nil +} + +// maxTranscriptCopyAttempts bounds how often pushMessages copies a transcript +// that local sync keeps rewriting. +const maxTranscriptCopyAttempts = 3 + +// localTranscriptRevision returns a session's local transcript revision, or +// "" once the session is gone locally. +func (s *Sync) localTranscriptRevision(ctx context.Context, sessionID string) (string, error) { + rev, err := s.local.TranscriptRevision(ctx, sessionID) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + return rev, err +} + +// copySessionMessages replaces a session's PostgreSQL messages, tool calls, +// and tool result events with the local transcript. +func (s *Sync) copySessionMessages(ctx context.Context, tx *sql.Tx, sessionID string) (int, error) { if _, err := tx.ExecContext(ctx, ` DELETE FROM tool_result_events WHERE session_id = $1 @@ -3320,10 +3392,6 @@ func (s *Sync) pushMessages( "deleting pg messages: %w", err, ) } - if err := s.replaceUsageEvents(ctx, tx, sessionID); err != nil { - return 0, err - } - count := 0 startOrdinal := 0 for { @@ -3367,14 +3435,10 @@ func (s *Sync) pushMessages( } count += len(msgs) startOrdinal = nextOrdinal + if s.afterMessagePage != nil { + s.afterMessagePage() + } } - - if err := restorePinnedMessages( - ctx, tx, sessionID, savedPins, - ); err != nil { - return count, err - } - return count, nil } diff --git a/internal/postgres/push_transcript_pgtest_test.go b/internal/postgres/push_transcript_pgtest_test.go new file mode 100644 index 0000000000..36ccacbc60 --- /dev/null +++ b/internal/postgres/push_transcript_pgtest_test.go @@ -0,0 +1,69 @@ +//go:build pgtest + +package postgres + +import ( + "fmt" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/storage" +) + +// Local sync can rewrite a transcript while a push copies it page by page. +// A pinned message that moves into a page the push already copied must keep +// its pin. +func TestPushKeepsPinWhenTranscriptShiftsDuringCopy(t *testing.T) { + const id = "claude:pin-shift" + const schema = "agentsview_pin_shift_test" + const total = db.MaxMessageLimit + 500 + const dropped = 300 + ctx := t.Context() + pgURL := testPGURL(t) + cleanNamedPGSchema(t, pgURL, schema) + t.Cleanup(func() { cleanNamedPGSchema(t, pgURL, schema) }) + local := testDB(t) + transcript := func(from int) []db.Message { + msgs := make([]db.Message, 0, total-from) + for i := from; i < total; i++ { + msgs = append(msgs, db.Message{ + SessionID: id, Ordinal: i - from, Role: "user", + Content: fmt.Sprintf("message %d", i), SourceUUID: fmt.Sprintf("uuid-%d", i), + Timestamp: "2026-09-25T12:00:00Z", + }) + } + return msgs + } + require.NoError(t, local.UpsertSession(ctx, db.Session{ + ID: id, Agent: "claude", Project: "sample", Machine: "machine", + MessageCount: total, CreatedAt: "2026-09-25T12:00:00Z", + })) + require.NoError(t, local.InsertMessages(ctx, transcript(0))) + syncer, err := New(pgURL, schema, local, "machine", true, storage.PusherOptions{}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, syncer.Close()) }) + result, err := syncer.Push(ctx, true, nil) + require.NoError(t, err) + require.Zero(t, result.Errors) + store := &Store{pg: syncer.pg} + pinned := db.MaxMessageLimit + 200 + _, err = store.PinMessage(ctx, id, int64(pinned), new("remote note")) + require.NoError(t, err) + + // Drop the first messages once the first page is copied, so the pinned + // message moves into the range the push has already read. + syncer.afterMessagePage = sync.OnceFunc(func() { + require.NoError(t, local.ReplaceSessionMessages(ctx, id, transcript(dropped))) + }) + result, err = syncer.Push(ctx, true, nil) + require.NoError(t, err) + require.Zero(t, result.Errors) + + pins, err := store.ListPinnedMessages(ctx, id, "") + require.NoError(t, err) + require.Len(t, pins, 1, "the push dropped a pin whose message moved during the copy") + assert.Equal(t, pinned-dropped, pins[0].Ordinal) +} diff --git a/internal/postgres/sync.go b/internal/postgres/sync.go index 041471b6be..4ce24178a4 100644 --- a/internal/postgres/sync.go +++ b/internal/postgres/sync.go @@ -302,6 +302,8 @@ type Sync struct { pushBatchSize int // afterPushBatch is a test hook called after each session batch. afterPushBatch func() + // afterMessagePage is a test hook called after each copied message page. + afterMessagePage func() closeOnce sync.Once closeErr error From 9a4d76fac7ae80a53c8dd7549721e602abac9494 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:53:29 -0400 Subject: [PATCH 09/11] fix(push): keep pushing a session that grows while it is copied The previous commit copied a session again whenever its transcript revision changed during the copy, and failed the session after three copies. Every local write bumps that revision, including appends from an agent that is still working. A long session in active use could change during all three copies, so the push skipped it and counted it as an error, where main pushed it. Only one gap from a changed transcript is permanent: a pin whose message moved into a page the copy had already read, because the pin restore then deletes it. The push now copies again only when the revision changed and a saved pin no longer finds its message. Any other gap needs no retry, because the local change stamps the session after this push's cutoff and the next push copies it again. The error for a session that keeps changing, and the log line for kept Codex threads, now use plain words. The error no longer repeats the session ID that the caller's log line already shows; the Codex page caller adds the page ID itself. Not changed: roborev-ci on e055588 again reported that a canceled pricing refresh waits for the archive writer to restore the old refresh time, which can push daemon shutdown past its timeout. This is the finding already ruled not an issue on d9ecc88; the wait lasts at most one sync write. Co-Authored-By: Claude Opus 5.5 --- internal/postgres/codex_page_pins.go | 2 +- internal/postgres/push.go | 35 +++++++++-- .../postgres/push_transcript_pgtest_test.go | 59 +++++++++++++++++++ 3 files changed, 89 insertions(+), 7 deletions(-) diff --git a/internal/postgres/codex_page_pins.go b/internal/postgres/codex_page_pins.go index 61698bf6b8..60a4840bcc 100644 --- a/internal/postgres/codex_page_pins.go +++ b/internal/postgres/codex_page_pins.go @@ -173,7 +173,7 @@ func (s *Sync) migrateCodexPages( return fmt.Errorf("publishing retained Codex page: %w", err) } if _, err := s.pushMessages(ctx, tx, pageID, true, nil, nil); err != nil { - return err + return fmt.Errorf("publishing retained Codex page %s: %w", pageID, err) } if _, err := s.pushSecretFindings(ctx, tx, pageID); err != nil { return err diff --git a/internal/postgres/push.go b/internal/postgres/push.go index fc5577b04e..54aca13fb6 100644 --- a/internal/postgres/push.go +++ b/internal/postgres/push.go @@ -2208,7 +2208,7 @@ func (s *Sync) reconcilePGMissingSessions( return err } if len(kept) > 0 { - log.Printf("postgres push: kept %d Codex thread(s) whose curation did not move to their page", len(kept)) + log.Printf("postgres push: kept %d older Codex thread(s) that still hold pins, stars, or titles", len(kept)) missing = slices.DeleteFunc(missing, func(id string) bool { return slices.Contains(kept, id) }) } // Recheck ownership and scope, and clear vectors only for deleted rows. @@ -3314,9 +3314,11 @@ func (s *Sync) pushMessages( return 0, err } // Local sync can rewrite this transcript while the copy reads it page by - // page. A pinned message that moves across a page boundary would be + // page. A pinned message that moves into a page the copy already read is // missed, and restoring the pins would then drop its pin, so copy again - // whenever the transcript changed during the copy. + // when the transcript changed and a pin lost its message. Other gaps from + // a changed transcript need no retry: the change stamps the session after + // this push's cutoff, so the next push copies it again. var count int for attempt := 1; ; attempt++ { before, err := s.localTranscriptRevision(ctx, sessionID) @@ -3334,10 +3336,16 @@ func (s *Sync) pushMessages( if before == after { break } + found, err := pinsFindMessages(ctx, tx, sessionID, savedPins) + if err != nil { + return count, err + } + if found { + break + } if attempt == maxTranscriptCopyAttempts { - return count, fmt.Errorf( - "pushMessages %s: the transcript changed during %d copies; push again", - sessionID, attempt, + return count, errors.New( + "the session changed while it was being copied; push again once it stops changing", ) } } @@ -3355,6 +3363,21 @@ func (s *Sync) pushMessages( // that local sync keeps rewriting. const maxTranscriptCopyAttempts = 3 +// pinsFindMessages reports whether every snapshotted pin still identifies a +// message among the rows copied for sessionID. +func pinsFindMessages( + ctx context.Context, tx *sql.Tx, sessionID string, + pins []savedPostgresPin, +) (bool, error) { + for _, pin := range pins { + _, _, ok, err := resolvePinnedMessageTarget(ctx, tx, sessionID, pin) + if err != nil || !ok { + return false, err + } + } + return true, nil +} + // localTranscriptRevision returns a session's local transcript revision, or // "" once the session is gone locally. func (s *Sync) localTranscriptRevision(ctx context.Context, sessionID string) (string, error) { diff --git a/internal/postgres/push_transcript_pgtest_test.go b/internal/postgres/push_transcript_pgtest_test.go index 36ccacbc60..c190485370 100644 --- a/internal/postgres/push_transcript_pgtest_test.go +++ b/internal/postgres/push_transcript_pgtest_test.go @@ -67,3 +67,62 @@ func TestPushKeepsPinWhenTranscriptShiftsDuringCopy(t *testing.T) { require.Len(t, pins, 1, "the push dropped a pin whose message moved during the copy") assert.Equal(t, pinned-dropped, pins[0].Ordinal) } + +// An agent that keeps writing to a long session changes its transcript during +// every copy. Appends cannot move a pinned message, so the push must still +// copy the session and keep its pin. +func TestPushCopiesSessionThatGrowsDuringEveryCopy(t *testing.T) { + const id = "claude:pin-grow" + const schema = "agentsview_pin_grow_test" + const total = db.MaxMessageLimit + 500 + ctx := t.Context() + pgURL := testPGURL(t) + cleanNamedPGSchema(t, pgURL, schema) + t.Cleanup(func() { cleanNamedPGSchema(t, pgURL, schema) }) + local := testDB(t) + message := func(ordinal int) db.Message { + return db.Message{ + SessionID: id, Ordinal: ordinal, Role: "user", + Content: fmt.Sprintf("message %d", ordinal), SourceUUID: fmt.Sprintf("uuid-%d", ordinal), + Timestamp: "2026-09-25T12:00:00Z", + } + } + msgs := make([]db.Message, 0, total) + for i := range total { + msgs = append(msgs, message(i)) + } + require.NoError(t, local.UpsertSession(ctx, db.Session{ + ID: id, Agent: "claude", Project: "sample", Machine: "machine", + MessageCount: total, CreatedAt: "2026-09-25T12:00:00Z", + })) + require.NoError(t, local.InsertMessages(ctx, msgs)) + syncer, err := New(pgURL, schema, local, "machine", true, storage.PusherOptions{}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, syncer.Close()) }) + result, err := syncer.Push(ctx, true, nil) + require.NoError(t, err) + require.Zero(t, result.Errors) + store := &Store{pg: syncer.pg} + pinned := db.MaxMessageLimit + 200 + _, err = store.PinMessage(ctx, id, int64(pinned), new("remote note")) + require.NoError(t, err) + + // Each copy reads two pages. Append after the first page of every copy, + // so the revision changes during each one. + next, pages := total, 0 + syncer.afterMessagePage = func() { + pages++ + if pages%2 == 1 { + require.NoError(t, local.InsertMessages(ctx, []db.Message{message(next)})) + next++ + } + } + result, err = syncer.Push(ctx, true, nil) + require.NoError(t, err) + require.Zero(t, result.Errors, "the push gave up on a session that only grew") + + pins, err := store.ListPinnedMessages(ctx, id, "") + require.NoError(t, err) + require.Len(t, pins, 1) + assert.Equal(t, pinned, pins[0].Ordinal) +} From c37633ce8c3a85b796287d3170cdf0f06a49c907 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Tue, 20 Oct 2026 07:59:43 -0400 Subject: [PATCH 10/11] fix(push): copy each PostgreSQL transcript from one SQLite snapshot Local sync keeps writing while a push copies a transcript 1,000 messages at a time. The last two commits guarded the pins with a copy-again loop that compared the transcript revision before and after the copy. Reviews of that loop found three more ways to break it: - A Codex page migration restores the old thread's pins onto the page after the copy, and the loop checked only the page's own pins, so a rewrite during the copy could still drop a thread pin and its note. - The copy read pages until it got an empty one, so a session that gained a message before every final short page kept the copy, the PostgreSQL transaction, and the daemon's push turn going. - A pin whose message was deleted long ago never resolves, so an active session with such a pin failed all three copies and was not pushed. Each of these comes from reading one transcript across several moments. The copy now reads every page inside one read-only SQLite transaction, which sees a single fixed transcript. Every pin finds the message it had when the copy started, the copy ends at that transcript's last message, and the retry loop and its error are gone. Writes that land after the snapshot carry a stamp after the push's cutoff, so the next push copies them. A long copy keeps one reader connection and its WAL snapshot open for its duration. The archive's other readers and its writer are not blocked; SQLite only defers checkpointing past that snapshot until the copy ends. Not changed: roborev-ci on 9a4d76f again reported that a canceled pricing refresh waits for the archive writer to restore the old refresh time. This is the finding already ruled not an issue on d9ecc88 and e055588. Co-Authored-By: Claude Opus 5.5 --- internal/db/db_test.go | 36 ++++++++ internal/db/messages.go | 50 ++++++++++- internal/postgres/codex_page_pins.go | 2 +- internal/postgres/push.go | 83 ++++--------------- .../postgres/push_transcript_pgtest_test.go | 51 ++++++++---- 5 files changed, 138 insertions(+), 84 deletions(-) diff --git a/internal/db/db_test.go b/internal/db/db_test.go index 1ace90f61e..3f9094e9da 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -3920,6 +3920,42 @@ func TestGetMessagesReturnsToolCalls(t *testing.T) { assert.Equal(t, 42, tc.ResultContentLength, "ResultContentLength =") } +// A message snapshot keeps reading the transcript it started with, tool calls +// included, while sync rewrites the session. +func TestMessageSnapshotIgnoresLaterWrites(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s1", "proj") + insertMessages(t, d, + Message{ + SessionID: "s1", Ordinal: 0, Role: "assistant", Content: "first", + Timestamp: tsZero, HasToolUse: true, + ToolCalls: []ToolCall{{SessionID: "s1", ToolName: "Read", Category: "Read", ToolUseID: "toolu_1"}}, + }, + Message{SessionID: "s1", Ordinal: 1, Role: "user", Content: "second", Timestamp: tsZero}, + ) + snapshot, err := d.BeginMessageSnapshot(t.Context()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, snapshot.Close()) }) + first, err := snapshot.GetMessages(t.Context(), "s1", 0, 1, true) + require.NoError(t, err) + require.Len(t, first, 1) + require.Len(t, first[0].ToolCalls, 1) + + require.NoError(t, d.ReplaceSessionMessages(t.Context(), "s1", []Message{ + {SessionID: "s1", Ordinal: 0, Role: "user", Content: "rewritten", Timestamp: tsZero}, + {SessionID: "s1", Ordinal: 1, Role: "user", Content: "rewritten too", Timestamp: tsZero}, + {SessionID: "s1", Ordinal: 2, Role: "user", Content: "added", Timestamp: tsZero}, + })) + + rest, err := snapshot.GetMessages(t.Context(), "s1", 1, 100, true) + require.NoError(t, err) + require.Len(t, rest, 1) + assert.Equal(t, "second", rest[0].Content) + current, err := d.GetMessages(t.Context(), "s1", 0, 100, true) + require.NoError(t, err) + assert.Len(t, current, 3) +} + func TestToolCallResultContent(t *testing.T) { database := testDB(t) sess := Session{ diff --git a/internal/db/messages.go b/internal/db/messages.go index b73bd0647e..b0a8e5384a 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -567,6 +567,52 @@ func (m Message) TokenPresence() (bool, bool) { func (db *DB) GetMessages( ctx context.Context, sessionID string, from, limit int, asc bool, +) ([]Message, error) { + return getMessagesWith(ctx, db.getReader(), sessionID, from, limit, asc) +} + +// MessageSnapshot reads messages from one read-only transaction, so every +// page sees the same transcript while sync keeps writing the archive. +type MessageSnapshot struct { + tx *sql.Tx +} + +// BeginMessageSnapshot starts a MessageSnapshot. Close it when done. +func (db *DB) BeginMessageSnapshot(ctx context.Context) (*MessageSnapshot, error) { + db.connMu.RLock() + reader := db.reader.Load() + if reader == nil { + db.connMu.RUnlock() + return nil, errors.New("database is closed") + } + tx, err := reader.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + db.connMu.RUnlock() + if err != nil { + return nil, fmt.Errorf("beginning message snapshot: %w", err) + } + return &MessageSnapshot{tx: tx}, nil +} + +// GetMessages works like DB.GetMessages, reading from the snapshot. +func (s *MessageSnapshot) GetMessages( + ctx context.Context, + sessionID string, from, limit int, asc bool, +) ([]Message, error) { + return getMessagesWith(ctx, s.tx, sessionID, from, limit, asc) +} + +// Close ends the snapshot. A snapshot whose context already ended has +// nothing left to release. +func (s *MessageSnapshot) Close() error { + if err := s.tx.Rollback(); !errors.Is(err, sql.ErrTxDone) { + return err + } + return nil +} + +func getMessagesWith( + ctx context.Context, q messageRowsQuerier, + sessionID string, from, limit int, asc bool, ) ([]Message, error) { if limit <= 0 || limit > MaxMessageLimit { limit = DefaultMessageLimit @@ -586,7 +632,7 @@ func (db *DB) GetMessages( ORDER BY ordinal %s LIMIT ?`, selectMessageCols, op, dir) - rows, err := db.getReader().QueryContext( + rows, err := q.QueryContext( ctx, query, sessionID, from, limit, ) if err != nil { @@ -597,7 +643,7 @@ func (db *DB) GetMessages( if err != nil { return nil, err } - if err := db.attachToolCalls(ctx, msgs); err != nil { + if err := attachToolCallsWithQuerier(ctx, q, msgs); err != nil { return nil, err } return msgs, nil diff --git a/internal/postgres/codex_page_pins.go b/internal/postgres/codex_page_pins.go index 60a4840bcc..61698bf6b8 100644 --- a/internal/postgres/codex_page_pins.go +++ b/internal/postgres/codex_page_pins.go @@ -173,7 +173,7 @@ func (s *Sync) migrateCodexPages( return fmt.Errorf("publishing retained Codex page: %w", err) } if _, err := s.pushMessages(ctx, tx, pageID, true, nil, nil); err != nil { - return fmt.Errorf("publishing retained Codex page %s: %w", pageID, err) + return err } if _, err := s.pushSecretFindings(ctx, tx, pageID); err != nil { return err diff --git a/internal/postgres/push.go b/internal/postgres/push.go index 54aca13fb6..5f3af175ea 100644 --- a/internal/postgres/push.go +++ b/internal/postgres/push.go @@ -3313,41 +3313,9 @@ func (s *Sync) pushMessages( if err := s.replaceUsageEvents(ctx, tx, sessionID); err != nil { return 0, err } - // Local sync can rewrite this transcript while the copy reads it page by - // page. A pinned message that moves into a page the copy already read is - // missed, and restoring the pins would then drop its pin, so copy again - // when the transcript changed and a pin lost its message. Other gaps from - // a changed transcript need no retry: the change stamps the session after - // this push's cutoff, so the next push copies it again. - var count int - for attempt := 1; ; attempt++ { - before, err := s.localTranscriptRevision(ctx, sessionID) - if err != nil { - return 0, err - } - count, err = s.copySessionMessages(ctx, tx, sessionID) - if err != nil { - return count, err - } - after, err := s.localTranscriptRevision(ctx, sessionID) - if err != nil { - return count, err - } - if before == after { - break - } - found, err := pinsFindMessages(ctx, tx, sessionID, savedPins) - if err != nil { - return count, err - } - if found { - break - } - if attempt == maxTranscriptCopyAttempts { - return count, errors.New( - "the session changed while it was being copied; push again once it stops changing", - ) - } + count, err := s.copySessionMessages(ctx, tx, sessionID) + if err != nil { + return count, err } if err := restorePinnedMessages( @@ -3359,38 +3327,20 @@ func (s *Sync) pushMessages( return count, nil } -// maxTranscriptCopyAttempts bounds how often pushMessages copies a transcript -// that local sync keeps rewriting. -const maxTranscriptCopyAttempts = 3 - -// pinsFindMessages reports whether every snapshotted pin still identifies a -// message among the rows copied for sessionID. -func pinsFindMessages( +// copySessionMessages replaces a session's PostgreSQL messages, tool calls, +// and tool result events with the local transcript. It reads every page from +// one SQLite snapshot. Local sync keeps writing during a push, and a rewrite +// between two page reads could move a pinned message into a range already +// copied, so restoring the pins would drop that pin. Messages that arrive +// after the snapshot go out with the next push. +func (s *Sync) copySessionMessages( ctx context.Context, tx *sql.Tx, sessionID string, - pins []savedPostgresPin, -) (bool, error) { - for _, pin := range pins { - _, _, ok, err := resolvePinnedMessageTarget(ctx, tx, sessionID, pin) - if err != nil || !ok { - return false, err - } - } - return true, nil -} - -// localTranscriptRevision returns a session's local transcript revision, or -// "" once the session is gone locally. -func (s *Sync) localTranscriptRevision(ctx context.Context, sessionID string) (string, error) { - rev, err := s.local.TranscriptRevision(ctx, sessionID) - if errors.Is(err, sql.ErrNoRows) { - return "", nil +) (count int, err error) { + snapshot, err := s.local.BeginMessageSnapshot(ctx) + if err != nil { + return 0, err } - return rev, err -} - -// copySessionMessages replaces a session's PostgreSQL messages, tool calls, -// and tool result events with the local transcript. -func (s *Sync) copySessionMessages(ctx context.Context, tx *sql.Tx, sessionID string) (int, error) { + defer func() { err = errors.Join(err, snapshot.Close()) }() if _, err := tx.ExecContext(ctx, ` DELETE FROM tool_result_events WHERE session_id = $1 @@ -3415,10 +3365,9 @@ func (s *Sync) copySessionMessages(ctx context.Context, tx *sql.Tx, sessionID st "deleting pg messages: %w", err, ) } - count := 0 startOrdinal := 0 for { - msgs, err := s.local.GetMessages( + msgs, err := snapshot.GetMessages( ctx, sessionID, startOrdinal, db.MaxMessageLimit, true, ) diff --git a/internal/postgres/push_transcript_pgtest_test.go b/internal/postgres/push_transcript_pgtest_test.go index c190485370..ac72fc0259 100644 --- a/internal/postgres/push_transcript_pgtest_test.go +++ b/internal/postgres/push_transcript_pgtest_test.go @@ -65,16 +65,29 @@ func TestPushKeepsPinWhenTranscriptShiftsDuringCopy(t *testing.T) { pins, err := store.ListPinnedMessages(ctx, id, "") require.NoError(t, err) require.Len(t, pins, 1, "the push dropped a pin whose message moved during the copy") + assert.Equal(t, pinned, pins[0].Ordinal, "the push copied a transcript that changed after it started") + + // The next push copies the rewritten transcript, and the pin follows its + // message. + syncer.afterMessagePage = nil + result, err = syncer.Push(ctx, true, nil) + require.NoError(t, err) + require.Zero(t, result.Errors) + pins, err = store.ListPinnedMessages(ctx, id, "") + require.NoError(t, err) + require.Len(t, pins, 1) assert.Equal(t, pinned-dropped, pins[0].Ordinal) } -// An agent that keeps writing to a long session changes its transcript during -// every copy. Appends cannot move a pinned message, so the push must still -// copy the session and keep its pin. -func TestPushCopiesSessionThatGrowsDuringEveryCopy(t *testing.T) { +// An agent that keeps writing to a long session adds messages while the push +// copies it. The copy must stop at the transcript it started with, keep the +// pin whose message is still there, and drop only the pin whose message is +// gone, as it would without the writes. +func TestPushCopiesSessionThatGrowsDuringTheCopy(t *testing.T) { const id = "claude:pin-grow" const schema = "agentsview_pin_grow_test" const total = db.MaxMessageLimit + 500 + const orphaned = 5 ctx := t.Context() pgURL := testPGURL(t) cleanNamedPGSchema(t, pgURL, schema) @@ -87,15 +100,18 @@ func TestPushCopiesSessionThatGrowsDuringEveryCopy(t *testing.T) { Timestamp: "2026-09-25T12:00:00Z", } } - msgs := make([]db.Message, 0, total) - for i := range total { - msgs = append(msgs, message(i)) + transcript := func() []db.Message { + msgs := make([]db.Message, 0, total) + for i := range total { + msgs = append(msgs, message(i)) + } + return msgs } require.NoError(t, local.UpsertSession(ctx, db.Session{ ID: id, Agent: "claude", Project: "sample", Machine: "machine", MessageCount: total, CreatedAt: "2026-09-25T12:00:00Z", })) - require.NoError(t, local.InsertMessages(ctx, msgs)) + require.NoError(t, local.InsertMessages(ctx, transcript())) syncer, err := New(pgURL, schema, local, "machine", true, storage.PusherOptions{}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, syncer.Close()) }) @@ -106,16 +122,20 @@ func TestPushCopiesSessionThatGrowsDuringEveryCopy(t *testing.T) { pinned := db.MaxMessageLimit + 200 _, err = store.PinMessage(ctx, id, int64(pinned), new("remote note")) require.NoError(t, err) + _, err = store.PinMessage(ctx, id, orphaned, new("orphaned note")) + require.NoError(t, err) + // Replace the orphaned pin's message, so that pin has nothing to match. + rewritten := transcript() + rewritten[orphaned].SourceUUID = "uuid-replaced" + rewritten[orphaned].Content = "replaced message" + require.NoError(t, local.ReplaceSessionMessages(ctx, id, rewritten)) - // Each copy reads two pages. Append after the first page of every copy, - // so the revision changes during each one. next, pages := total, 0 syncer.afterMessagePage = func() { pages++ - if pages%2 == 1 { - require.NoError(t, local.InsertMessages(ctx, []db.Message{message(next)})) - next++ - } + require.LessOrEqual(t, pages, 3, "the copy kept following new messages") + require.NoError(t, local.InsertMessages(ctx, []db.Message{message(next)})) + next++ } result, err = syncer.Push(ctx, true, nil) require.NoError(t, err) @@ -125,4 +145,7 @@ func TestPushCopiesSessionThatGrowsDuringEveryCopy(t *testing.T) { require.NoError(t, err) require.Len(t, pins, 1) assert.Equal(t, pinned, pins[0].Ordinal) + tail, err := store.GetMessages(ctx, id, total-1, 10, true) + require.NoError(t, err) + assert.Len(t, tail, 1, "the copy included messages written after it started") } From 25d51562159a0acd3fbed1ee8678fd3ef708687e Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:48:33 -0400 Subject: [PATCH 11/11] test(push): check that an incremental push picks up a mid-copy rewrite The snapshot copy leaves a transcript rewrite that lands during a push for the next push. That depends on the rewrite stamping the session after the push's cutoff. The test's follow-up push was a full push, which copies every session anyway, so it would still pass if that stamp or the cutoff broke. It now runs an incremental push and fails when the push records its cutoff after the copy instead of before. BeginMessageSnapshot now starts its transaction through the reader handle like the archive's other read snapshots, which already hold the connection lock around BeginTx. Not changed: - roborev-ci on c37633c again reported that a canceled pricing refresh waits for the archive writer to restore the old refresh time. This is the finding already ruled not an issue on earlier heads. - A failed copy during a Codex page migration still returns its error without naming the page, as main does. Co-Authored-By: Claude Opus 5.5 --- internal/db/messages.go | 9 +-------- internal/postgres/push_transcript_pgtest_test.go | 6 +++--- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/internal/db/messages.go b/internal/db/messages.go index b0a8e5384a..7959d98e85 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -579,14 +579,7 @@ type MessageSnapshot struct { // BeginMessageSnapshot starts a MessageSnapshot. Close it when done. func (db *DB) BeginMessageSnapshot(ctx context.Context) (*MessageSnapshot, error) { - db.connMu.RLock() - reader := db.reader.Load() - if reader == nil { - db.connMu.RUnlock() - return nil, errors.New("database is closed") - } - tx, err := reader.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) - db.connMu.RUnlock() + tx, err := db.getReader().BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) if err != nil { return nil, fmt.Errorf("beginning message snapshot: %w", err) } diff --git a/internal/postgres/push_transcript_pgtest_test.go b/internal/postgres/push_transcript_pgtest_test.go index ac72fc0259..6646eb65ff 100644 --- a/internal/postgres/push_transcript_pgtest_test.go +++ b/internal/postgres/push_transcript_pgtest_test.go @@ -67,10 +67,10 @@ func TestPushKeepsPinWhenTranscriptShiftsDuringCopy(t *testing.T) { require.Len(t, pins, 1, "the push dropped a pin whose message moved during the copy") assert.Equal(t, pinned, pins[0].Ordinal, "the push copied a transcript that changed after it started") - // The next push copies the rewritten transcript, and the pin follows its - // message. + // The rewrite landed after the push's cutoff, so the next incremental push + // copies it, and the pin follows its message. syncer.afterMessagePage = nil - result, err = syncer.Push(ctx, true, nil) + result, err = syncer.Push(ctx, false, nil) require.NoError(t, err) require.Zero(t, result.Errors) pins, err = store.ListPinnedMessages(ctx, id, "")