From e5bf55339f8aaf05dd294228bfc50fab8363ab70 Mon Sep 17 00:00:00 2001 From: x Date: Fri, 9 Oct 2026 23:02:42 +0000 Subject: [PATCH 01/11] feat(ci): group a run's steps into trace spans and name CI's own steps Jobs, commands and extra machines become spans, CI's steps are named for what they do and marked as CI's work. A job's `name` sets its span name. Co-Authored-By: Claude Opus 5.5 --- .changeset/ci-trace-spans.md | 5 + packages/ci/src/cache/cache.ts | 28 +- packages/ci/src/checkout/changed.ts | 3 +- packages/ci/src/checkout/wait.ts | 4 +- packages/ci/src/github/checks.test.ts | 3 +- packages/ci/src/github/checks.ts | 161 +++-- packages/ci/src/github/github.test.ts | 2 +- packages/ci/src/github/helpers.ts | 114 ++-- packages/ci/src/machine/command.ts | 172 ++++-- packages/ci/src/machine/from.ts | 15 +- packages/ci/src/machine/machine.ts | 150 +++-- packages/ci/src/machine/spans.test.ts | 583 ++++++++++++++++++ packages/ci/src/pipeline/README.md | 2 + .../ci/src/pipeline/cacheGuarantees.test.ts | 12 + packages/ci/src/pipeline/durable.ts | 3 +- packages/ci/src/pipeline/job.ts | 46 +- packages/ci/src/pipeline/metadata.test.ts | 9 + packages/ci/src/pipeline/metadata.ts | 3 +- packages/ci/src/pipeline/names.ts | 160 +++++ packages/ci/src/pipeline/pipeline.ts | 102 +-- packages/ci/src/pipeline/scope.ts | 59 +- packages/ci/src/pipeline/spans.test.ts | 146 +++++ packages/ci/src/pipeline/spans.ts | 70 +++ packages/ci/src/report.ts | 5 +- packages/ci/src/testing/README.md | 1 + packages/ci/src/testing/runFunction.ts | 43 +- packages/ci/src/testing/setup.ts | 11 + packages/ci/src/testing/spanStub.ts | 185 ++++++ packages/ci/src/types.ts | 4 +- packages/ci/src/units.test.ts | 8 + packages/ci/vitest.config.ts | 1 + 31 files changed, 1825 insertions(+), 285 deletions(-) create mode 100644 .changeset/ci-trace-spans.md create mode 100644 packages/ci/src/machine/spans.test.ts create mode 100644 packages/ci/src/pipeline/names.ts create mode 100644 packages/ci/src/pipeline/spans.test.ts create mode 100644 packages/ci/src/pipeline/spans.ts create mode 100644 packages/ci/src/testing/setup.ts create mode 100644 packages/ci/src/testing/spanStub.ts diff --git a/.changeset/ci-trace-spans.md b/.changeset/ci-trace-spans.md new file mode 100644 index 000000000..d3ef514f5 --- /dev/null +++ b/.changeset/ci-trace-spans.md @@ -0,0 +1,5 @@ +--- +"@inngest/ci": minor +--- + +Group a run's steps into trace spans (one per job, one per command, one per extra machine), name CI's own steps for what they do, and mark them as CI's work so a trace can tell them from yours. Give a job a `name` to change what its span is called. diff --git a/packages/ci/src/cache/cache.ts b/packages/ci/src/cache/cache.ts index 2d5d99fb2..d53d856b0 100644 --- a/packages/ci/src/cache/cache.ts +++ b/packages/ci/src/cache/cache.ts @@ -11,6 +11,7 @@ */ import { tagStep } from "../pipeline/metadata.ts"; +import { ciStep, traceName } from "../pipeline/names.ts"; import type { CiJobScope, CiRunScope } from "../pipeline/scope.ts"; import { countApi, rootRunIdOf, scopeSeparator } from "../pipeline/scope.ts"; import type { @@ -255,7 +256,7 @@ export interface CachedSnapshot { export interface CacheTarget { /** The job's resolved key. */ ownKey: string; - /** The name the snapshot is given, in the scope the job writes. */ + /** The name a build gives the snapshot, in the scope the job writes. */ name: string; } @@ -282,10 +283,7 @@ export const cacheTarget = async ( countApi("cache"); const ownKey = (await run.step.run( - { - id: `${job.path}${scopeSeparator}cache:key`, - name: "cache:key", - }, + ciStep(`${job.path}${scopeSeparator}cache:key`, traceName.checkCache), async () => { await tagStep(run, { kind: "cache", job: job.path }); @@ -337,7 +335,7 @@ const isExpiring = (expiresAt: string | undefined): boolean => { return Number.isFinite(at) && at - Date.now() < expiryMarginMs; }; -/** How long a lookup waits for a snapshot another run is still taking. */ +/** How long a lookup waits for a snapshot another build is still taking. */ const readyWaitMs = 2 * 60 * 1000; const readyPollMs = 1000; @@ -387,7 +385,7 @@ const waitUntilReady = async ( /** * The newest snapshot with exactly this name, if it can be used: ready, or - * ready after waiting for the run taking it, and not about to expire. + * ready after waiting for the build taking it, and not about to expire. * * Any error is a miss, and only an exact name counts, since a server that * doesn't know names ignores the filter and lists every snapshot. @@ -464,10 +462,7 @@ export const lookupCache = async ( const { run } = scope; const found = (await run.step.run( - { - id: `${scope.path}${scopeSeparator}cache:lookup`, - name: "cache:lookup", - }, + ciStep(`${scope.path}${scopeSeparator}cache:lookup`, traceName.lookUpCache), async () => { await tagStep(run, { kind: "cache", job: scope.path }); @@ -521,10 +516,7 @@ export const lookupBeforeBuild = async ( exclude?: string, ): Promise => { const found = (await run.step.run( - { - id: `${job.stepPath}${scopeSeparator}lookup`, - name: "cache:lookup", - }, + ciStep(`${job.stepPath}${scopeSeparator}lookup`, traceName.lookUpCache), async () => { await tagStep(run, { kind: "cache", job: job.path }); @@ -555,7 +547,7 @@ export const resolveTakenName = async ( broken?: boolean, ): Promise<{ winner?: CachedSnapshot; cleared: boolean }> => { return (await run.step.run( - { id: stepId, name: "cache:name-taken" }, + ciStep(stepId, traceName.resolveCacheName), async () => { const winner = await findNamed(run, name, exclude); @@ -605,7 +597,7 @@ export const snapshotState = async ( snapshotId: string, ): Promise<"gone" | "creating" | "ready"> => { return (await run.step.run( - { id: stepId, name: "cache:snapshot-state" }, + ciStep(stepId, traceName.checkSnapshotState), async () => { try { const snapshot = (await snapshotsClient(run).get(snapshotId)) as @@ -637,7 +629,7 @@ export const deleteSnapshot = async ( ): Promise => { try { const result = (await run.step.run( - { id: stepId, name: "cache:delete" }, + ciStep(stepId, traceName.deleteBadSnapshot), async () => { try { const snapshot = await snapshotsClient(run).get(snapshotId); diff --git a/packages/ci/src/checkout/changed.ts b/packages/ci/src/checkout/changed.ts index 3e8433093..8fad40dce 100644 --- a/packages/ci/src/checkout/changed.ts +++ b/packages/ci/src/checkout/changed.ts @@ -6,6 +6,7 @@ */ import { CiUsageError } from "../errors.ts"; +import { traceName } from "../pipeline/names.ts"; import type { CiRunScope } from "../pipeline/scope.ts"; import { countApi, getRunScope } from "../pipeline/scope.ts"; import type { RepoContext } from "../types.ts"; @@ -91,7 +92,7 @@ export const changedFiles = async (): Promise => { const readChangedFiles = async (run: CiRunScope): Promise => { const files = (await run.step.run( - { id: "changed", name: "changed" }, + { id: "changed", name: traceName.findChangedFiles }, async () => { try { return await listChangedFiles(run.repo); diff --git a/packages/ci/src/checkout/wait.ts b/packages/ci/src/checkout/wait.ts index b91f9f762..526794143 100644 --- a/packages/ci/src/checkout/wait.ts +++ b/packages/ci/src/checkout/wait.ts @@ -81,7 +81,7 @@ export const waitForHttp = async ( return target; }, ["/bin/sh", "-c", script], - `waitForHttp ${url}`, + { label: `waitForHttp ${url}` }, ).timeout(`${timeoutMs + waitHeadroomMs}ms`); }; @@ -125,6 +125,6 @@ export const waitForPort = async ( return target; }, ["/bin/sh", "-c", script], - `waitForPort ${port}`, + { label: `waitForPort ${port}` }, ).timeout(`${timeoutMs + waitHeadroomMs}ms`); }; diff --git a/packages/ci/src/github/checks.test.ts b/packages/ci/src/github/checks.test.ts index ff38f43a2..4a6ae6077 100644 --- a/packages/ci/src/github/checks.test.ts +++ b/packages/ci/src/github/checks.test.ts @@ -42,12 +42,11 @@ const fakeRun = (overrides: Partial = {}): CiRunScope => { jobCalls: new Map(), machines: new Map(), snapshots: new Map(), + cached: new Map(), createdSnapshots: new Set(), - cachedSnapshots: new Map(), sandboxes: new Set(), summaries: [], counters: new Map(), - snapshotsUnavailable: false, warnings: [], pipelineSummaries: [], pipelineAnnotations: [], diff --git a/packages/ci/src/github/checks.ts b/packages/ci/src/github/checks.ts index 8f4fa5232..f6a36c54e 100644 --- a/packages/ci/src/github/checks.ts +++ b/packages/ci/src/github/checks.ts @@ -6,7 +6,9 @@ */ import { type StepTag, tagStep } from "../pipeline/metadata.ts"; +import { traceName } from "../pipeline/names.ts"; import type { CiRunScope } from "../pipeline/scope.ts"; +import { inGitHubSpan } from "../pipeline/scope.ts"; import type { CheckAnnotation, CheckConclusion } from "../types.ts"; import { errorMessage, formatDuration } from "../util.ts"; import type { GitHubProvider, Octokit } from "./auth.ts"; @@ -225,6 +227,17 @@ export const normaliseAnnotation = ( }; }; +/** Run a check update as a step in the run's GitHub span. */ +const githubStep = ( + run: CiRunScope, + step: { id: string; name: string }, + fn: () => Promise, +): Promise => { + return inGitHubSpan(run, () => { + return run.step.run(step, fn) as Promise; + }); +}; + /** * Build the reporter used by every pipeline run. * @@ -262,20 +275,17 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { run: CiRunScope, key: string, name: string, - stepId: string, + step: { id: string; name: string }, tag: StepTag, metadata?: () => Record, ) => { - const result = await run.step.run( - { id: stepId, name: stepId }, - async () => { - await tagStep(run, tag, metadata?.()); + const result = await githubStep(run, step, async () => { + await tagStep(run, tag, metadata?.()); - const started = await sink.start({ run, name, ...identity(run, key) }); + const started = await sink.start({ run, name, ...identity(run, key) }); - return { ...started, startedAt: Date.now() }; - }, - ); + return { ...started, startedAt: Date.now() }; + }); if (result?.id) { checkRunIds.set(idKey(run, key), result.id); @@ -288,32 +298,29 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { run: CiRunScope, key: string, name: string, - stepId: string, + step: { id: string; name: string }, result: () => CheckResult, tag: StepTag, metadata?: () => Record, ): Promise => { - const endedAt = await run.step.run( - { id: stepId, name: stepId }, - async () => { - await tagStep(run, tag, metadata?.()); + const endedAt = await githubStep(run, step, async () => { + await tagStep(run, tag, metadata?.()); - const read = result(); + const read = result(); - await sink.complete({ - run, - name, - ...identity(run, key), - conclusion: read.conclusion, - title: read.title, - summary: truncateSummary(read.summary ?? ""), - annotations: (read.annotations ?? []).map(normaliseAnnotation), - ...idFor(run, key), - }); + await sink.complete({ + run, + name, + ...identity(run, key), + conclusion: read.conclusion, + title: read.title, + summary: truncateSummary(read.summary ?? ""), + annotations: (read.annotations ?? []).map(normaliseAnnotation), + ...idFor(run, key), + }); - return Date.now(); - }, - ); + return Date.now(); + }); checkRunIds.delete(idKey(run, key)); @@ -330,7 +337,10 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { run, "pipeline", run.checkName, - `github › check:${run.checkName}:start`, + { + id: `github › check:${run.checkName}:start`, + name: traceName.createCheck(run.checkName), + }, { kind: "check" }, metadata, ); @@ -345,7 +355,10 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { run, "pipeline", run.checkName, - `github › check:${run.checkName}:complete`, + { + id: `github › check:${run.checkName}:complete`, + name: traceName.completeCheck(run.checkName), + }, result, { kind: "check" }, metadata, @@ -361,7 +374,10 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { run, jobPath, jobCheckName(run, jobPath, name), - `github › check:${jobPath}:start`, + { + id: `github › check:${jobPath}:start`, + name: traceName.report(jobPath, "started"), + }, { kind: "check", job: jobPath }, ); }, @@ -375,7 +391,10 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { run, jobPath, jobCheckName(run, jobPath, name), - `github › check:${jobPath}:complete`, + { + id: `github › check:${jobPath}:complete`, + name: traceName.report(jobPath, traceName.outcome(result.conclusion)), + }, () => { return result; }, @@ -394,31 +413,31 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return closing(read()); } - const stepId = "github › check:jobs:complete"; + const step = { + id: "github › check:jobs:complete", + name: traceName.report("jobs", "ended with the run"), + }; - const closed = (await run.step.run( - { id: stepId, name: stepId }, - async () => { - await tagStep(run, { kind: "check" }); - - const jobs = read(); - - for (const { jobPath, name, ...result } of jobs) { - await sink.complete({ - run, - name: jobCheckName(run, jobPath, name), - ...identity(run, jobPath), - conclusion: result.conclusion, - title: result.title, - summary: truncateSummary(result.summary ?? ""), - annotations: (result.annotations ?? []).map(normaliseAnnotation), - ...idFor(run, jobPath), - }); - } + const closed = await githubStep(run, step, async () => { + await tagStep(run, { kind: "check" }); - return closing(jobs); - }, - )) as Array>; + const jobs = read(); + + for (const { jobPath, name, ...result } of jobs) { + await sink.complete({ + run, + name: jobCheckName(run, jobPath, name), + ...identity(run, jobPath), + conclusion: result.conclusion, + title: result.title, + summary: truncateSummary(result.summary ?? ""), + annotations: (result.annotations ?? []).map(normaliseAnnotation), + ...idFor(run, jobPath), + }); + } + + return closing(jobs); + }); // Outside the step, so a replay also forgets the IDs its memoized start // steps put back. @@ -439,9 +458,15 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { jobPath === undefined ? run.checkName : jobCheckName(run, jobPath, name); - const stepId = `github › check:${key}:retry:${run.attempt}`; + const step = { + id: `github › check:${key}:retry:${run.attempt}`, + name: traceName.report( + jobPath ?? run.checkName, + traceName.retrying(run.attempt + 2, run.maxAttempts), + ), + }; - await run.step.run({ id: stepId, name: stepId }, async () => { + await githubStep(run, step, async () => { await tagStep( run, jobPath === undefined @@ -465,9 +490,15 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return; } - const stepId = `github › check:jobs:retry:${run.attempt}`; + const step = { + id: `github › check:jobs:retry:${run.attempt}`, + name: traceName.report( + "jobs", + traceName.retrying(run.attempt + 2, run.maxAttempts), + ), + }; - await run.step.run({ id: stepId, name: stepId }, async () => { + await githubStep(run, step, async () => { await tagStep(run, { kind: "check" }); for (const { jobPath, name } of read()) { @@ -490,10 +521,11 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { const message = errorMessage(error).split("\n")[0]; - await run.step.run( + await githubStep( + run, { id: `github › check:${jobPath}:attempt:${attempt}`, - name: `check:${jobPath}:attempt:${attempt}`, + name: traceName.report(jobPath, traceName.retrying(attempt + 1, of)), }, async () => { await sink.update?.({ @@ -527,9 +559,12 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { } const { parent } = run.build; - const stepId = `github › check:${parent.jobPath}:building`; + const step = { + id: `github › check:${parent.jobPath}:building`, + name: traceName.report(parent.jobPath, "building"), + }; - await run.step.run({ id: stepId, name: stepId }, async () => { + await githubStep(run, step, async () => { await sink.update?.({ // The check belongs to the pipeline that invoked this run. run: { ...run, pipelineId: parent.pipelineId }, diff --git a/packages/ci/src/github/github.test.ts b/packages/ci/src/github/github.test.ts index 35dca03b7..800265fd1 100644 --- a/packages/ci/src/github/github.test.ts +++ b/packages/ci/src/github/github.test.ts @@ -455,7 +455,7 @@ describe("github helpers", () => { { event: prEvent, resolveWait: (step) => { - return step.displayName?.includes("waitForChecks") + return step.displayName === "Wait for check: vercel" ? { name: "github/check_run.completed", data: { check_run: { name: "vercel", conclusion: "failure" } }, diff --git a/packages/ci/src/github/helpers.ts b/packages/ci/src/github/helpers.ts index 17867638e..1e6abc3e9 100644 --- a/packages/ci/src/github/helpers.ts +++ b/packages/ci/src/github/helpers.ts @@ -8,10 +8,13 @@ import { RetryAfterError } from "inngest"; import { CiUsageError } from "../errors.ts"; import { durablePath } from "../pipeline/durable.ts"; +import { traceName } from "../pipeline/names.ts"; +import type { CiRunScope } from "../pipeline/scope.ts"; import { countApi, getJobScope, getRunScope, + inGitHubSpan, requireRunScope, } from "../pipeline/scope.ts"; import type { CheckConclusion, Duration } from "../types.ts"; @@ -20,6 +23,14 @@ import type { Octokit } from "./auth.ts"; import { mapGitHubError, octokitForRun, rest } from "./rest.ts"; import { hasPermission, type Permission } from "./triggers.ts"; +/** + * Run a helper's steps in the job that called it, or in the run's GitHub span + * when no job did. + */ +const inHelperSpan = (run: CiRunScope, fn: () => T): T => { + return getJobScope() ? fn() : inGitHubSpan(run, fn); +}; + /** * Run a helper's body as a single step, so all the calls it makes retry * together. Inside the step, `github.rest` calls run directly. @@ -28,6 +39,8 @@ const helperStep = async ( helper: string, key: string, fn: () => Promise, + /** The step's name, which is the helper's method name by default. */ + name = traceName.githubHelper(helper), ): Promise => { const run = requireRunScope(`github.${helper}`); @@ -40,7 +53,9 @@ const helperStep = async ( const job = getJobScope(); const id = `${job ? `${job.path} › ` : ""}github › ${helper}:${key}`; - return run.step.run({ id, name: id }, fn) as Promise; + return inHelperSpan(run, () => { + return run.step.run({ id, name }, fn) as Promise; + }); }; /** Run `call` as a step when inside a run, or directly when outside one. */ @@ -458,30 +473,35 @@ export const canUser = async ( login: string, permission: Permission, ): Promise => { - return helperStep("canUser", `${login}:${permission}`, async () => { - try { - const result = await rest.repos.getCollaboratorPermissionLevel({ - username: login, - }); + return helperStep( + "canUser", + `${login}:${permission}`, + async () => { + try { + const result = await rest.repos.getCollaboratorPermissionLevel({ + username: login, + }); - return hasPermission(result.permission, permission); - } catch (error) { - const status = - (error as ErrorWithStatus).status ?? - (error as ErrorWithStatus).cause?.status; - - // A 403 or 404 here means "can't see it", which is the same as "no". - // Rate limits and server errors must retry, not become a "no". - if ( - (status === 403 || status === 404) && - !(error instanceof RetryAfterError) - ) { - return false; - } + return hasPermission(result.permission, permission); + } catch (error) { + const status = + (error as ErrorWithStatus).status ?? + (error as ErrorWithStatus).cause?.status; - throw error; - } - }); + // A 403 or 404 here means "can't see it", which is the same as "no". + // Rate limits and server errors must retry, not become a "no". + if ( + (status === 403 || status === 404) && + !(error instanceof RetryAfterError) + ) { + return false; + } + + throw error; + } + }, + traceName.canUser(login, permission), + ); }; /** @@ -537,18 +557,20 @@ export const waitForChecks = async (opts: { const waited = await Promise.all( missing.map(async (name) => { - const event = (await run.step.waitForEvent( - { - id: `github › waitForChecks:${name}`, - name: `waitForChecks:${name}`, - }, - { - event: "github/check_run.completed", - timeout: opts.timeout ?? "1h", - if: `async.data.check_run.name == ${JSON.stringify(name)} && async.data.check_run.head_sha == ${JSON.stringify(sha)}`, - }, + const event = (await inHelperSpan(run, () => { + return run.step.waitForEvent( + { + id: `github › waitForChecks:${name}`, + name: traceName.waitForCheck(name), + }, + { + event: "github/check_run.completed", + timeout: opts.timeout ?? "1h", + if: `async.data.check_run.name == ${JSON.stringify(name)} && async.data.check_run.head_sha == ${JSON.stringify(sha)}`, + }, + ); // biome-ignore lint/suspicious/noExplicitAny: event shape is the user's - )) as any; + })) as any; return [ name, @@ -597,18 +619,20 @@ export const waitForWorkflow = async (opts: { ? `async.data.workflow_run.workflow_id == ${opts.workflow}` : `async.data.workflow_run.path.endsWith(${JSON.stringify(`/${opts.workflow}`)})`; - const event = (await run.step.waitForEvent( - { - id: `github › waitForWorkflow:${opts.workflow}`, - name: `waitForWorkflow:${opts.workflow}`, - }, - { - event: "github/workflow_run.completed", - timeout: opts.timeout ?? "1h", - if: `async.data.workflow_run.head_sha == ${JSON.stringify(sha)} && ${workflowMatch}`, - }, + const event = (await inHelperSpan(run, () => { + return run.step.waitForEvent( + { + id: `github › waitForWorkflow:${opts.workflow}`, + name: traceName.waitForWorkflow(opts.workflow), + }, + { + event: "github/workflow_run.completed", + timeout: opts.timeout ?? "1h", + if: `async.data.workflow_run.head_sha == ${JSON.stringify(sha)} && ${workflowMatch}`, + }, + ); // biome-ignore lint/suspicious/noExplicitAny: event shape is the user's - )) as any; + })) as any; return (event?.data?.workflow_run?.conclusion ?? "timed_out") as | CheckConclusion diff --git a/packages/ci/src/machine/command.ts b/packages/ci/src/machine/command.ts index 46459e8d2..6e0a0f732 100644 --- a/packages/ci/src/machine/command.ts +++ b/packages/ci/src/machine/command.ts @@ -6,12 +6,14 @@ * @module */ +import { NonRetriableError, StepError } from "inngest"; import { getSandboxError } from "inngest/experimental"; import { CiUsageError, CommandFailedError, CommandTimeoutError, } from "../errors.ts"; +import { ciSpan, ciStep, traceName } from "../pipeline/names.ts"; import type { CiJobScope, MachineHandle } from "../pipeline/scope.ts"; import { countApi, @@ -20,6 +22,7 @@ import { requireJobScope, scopeSeparator, } from "../pipeline/scope.ts"; +import { inSpan } from "../pipeline/spans.ts"; import type { BackgroundProcess, Command, @@ -35,7 +38,7 @@ import { tail, truncateLabel, } from "../util.ts"; -import { ensureMachine } from "./machine.ts"; +import { ensureMachine, inMachineSpan } from "./machine.ts"; /** * Captured `commands.run` is capped at five minutes, so anything longer runs @@ -61,6 +64,9 @@ type SandboxProcess = any; interface CommandState { argv: string[]; + /** What the command reads as, when that isn't its argv, as for `$.sh`. */ + text?: string; + /** A name for the command, from `.as()` or the helper that made it. */ label?: string; env: Record; cwd?: string; @@ -165,11 +171,12 @@ export const buildShellString = ( return out; }; -/** The id and name of a step nested under `stepId`. */ -const subStep = (stepId: string, suffix: string) => { - const id = `${stepId}${scopeSeparator}${suffix}`; - - return { id, name: id }; +/** + * A step nested under `stepId`, named for what it does. It's CI's work: the + * command's span is yours, but how CI runs it isn't. + */ +const subStep = (stepId: string, suffix: string, name: string) => { + return ciStep(`${stepId}${scopeSeparator}${suffix}`, name); }; class CommandBuilder implements Command { @@ -261,11 +268,10 @@ class CommandBuilder implements Command { const stepId = this.stepId(scope); const machine = await ensureMachine(scope); - const process = await startProcess( - machine, - stepId, - this.spawnOptions(scope), - ); + // Each call back into the process re-enters the command's span. + const process = await this.inSpan(scope, stepId, () => { + return startProcess(machine, stepId, this.spawnOptions(scope)); + }); const argv = this.state.argv; const secrets = this.secretValues(scope); @@ -275,25 +281,36 @@ class CommandBuilder implements Command { return { id: process.id, - exited: async () => { - const polled = await pollUntilTerminal({ - scope, - machine, - process, - stepId, - nextWait, + exited: () => { + return this.inSpan(scope, stepId, async () => { + const polled = await pollUntilTerminal({ + scope, + machine, + process, + stepId, + nextWait, + }); + + return readResult({ process: polled.process, stepId, argv, secrets }); }); - - return readResult({ process: polled.process, stepId, argv, secrets }); }, kill: async (signal = 15) => { - await process.signal(subStep(stepId, "kill"), { signal }); + await this.inSpan(scope, stepId, () => { + return process.signal( + subStep(stepId, "kill", traceName.stopProcess), + { signal }, + ); + }); }, output: async (opts) => { - const output = await process.getOutput( - subStep(stepId, `output #${nextWait()}`), - { tailBytes: opts?.tailBytes ?? outputTailBytes }, - ); + const output = await this.inSpan(scope, stepId, () => { + return process.getOutput( + subStep(stepId, `output #${nextWait()}`, traceName.readOutput), + { + tailBytes: opts?.tailBytes ?? outputTailBytes, + }, + ); + }); const decoded = decodeChunks(output); @@ -313,8 +330,14 @@ class CommandBuilder implements Command { return this.started; } + /** What the command reads as: its `$.sh` script, or its arguments. */ + private commandText(): string { + return this.state.text ?? truncateLabel(this.state.argv.join(" ")); + } + + /** What the command's step IDs are built from. */ private labelText(): string { - return this.state.label ?? truncateLabel(this.state.argv.join(" ")); + return this.state.label ?? this.commandText(); } private stepId(scope: CiJobScope): string { @@ -348,6 +371,22 @@ class CommandBuilder implements Command { }); } + /** Run `fn` in the command's span, in its machine's span if it's an extra. */ + private inSpan(scope: CiJobScope, stepId: string, fn: () => T): T { + const span = { + id: stepId, + name: traceName.command(this.commandText(), this.state.label), + }; + + return inMachineSpan(scope, () => { + return inSpan(span, fn); + }); + } + + /** + * Each attempt runs in the command's span, and in a span of its own when the + * command has retries. A failed attempt's span ends in its failure. + */ private async runWithRetries(): Promise { const scope = this.getScope(); const stepId = this.stepId(scope); @@ -361,7 +400,24 @@ class CommandBuilder implements Command { scope.run.ci.hooks.commandStarted(scope, attemptInfo); - const result = await this.runOnce(scope, attemptId); + const runAttempt = async () => { + const result = await this.runOnce(scope, attemptId); + + if (result.exitCode !== 0 && !this.state.nothrow) { + await recordFailure(scope, attemptId, result.exitCode); + } + + return result; + }; + + const result = await this.inSpan(scope, stepId, () => { + return attempts === 1 + ? runAttempt() + : inSpan( + ciSpan(`attempt-${attempt}`, traceName.attempt(attempt)), + runAttempt, + ); + }); scope.run.ci.hooks.commandFinished(scope, attemptInfo, result); @@ -422,7 +478,7 @@ class CommandBuilder implements Command { try { const result = await machine.sandbox.commands.run( - { id: stepId, name: stepId }, + ciStep(stepId, traceName.runAndReadOutput), command, { ...options, timeout: timeoutMs }, ); @@ -473,7 +529,10 @@ class CommandBuilder implements Command { if (polled.timedOut) { await this.state.onTimeout?.(); - await process.signal(subStep(stepId, "timeout-kill"), { signal: 9 }); + await process.signal( + subStep(stepId, "timeout-kill", traceName.stopAfterTimeout), + { signal: 9 }, + ); throw this.timeoutError(scope); } @@ -500,6 +559,31 @@ const counter = (): (() => number) => { }; }; +/** + * Record that a run of a command failed, as a step that fails without retrying. + * The sandbox steps before it all succeed, since a non-zero exit is a result, + * so this step is what shows the run as failed in the trace. Its error is + * caught: the caller decides whether to retry or throw. + */ +const recordFailure = async ( + scope: CiJobScope, + stepId: string, + exitCode: number, +): Promise => { + try { + await scope.run.step.run( + subStep(stepId, "exit", traceName.exited(exitCode)), + () => { + throw new NonRetriableError(`exit ${exitCode}`); + }, + ); + } catch (error) { + if (!(error instanceof StepError)) { + throw error; + } + } +}; + /** * Start a managed process. Cloud can answer a start that succeeded with an * ambiguous error, in which case the process that did start is adopted @@ -520,7 +604,7 @@ const startProcess = async ( try { const process = await machine.sandbox.processes.start( - subStep(stepId, "start"), + subStep(stepId, "start", traceName.startProcess), options, ); @@ -570,7 +654,7 @@ const adoptAmbiguousStart = async ( } const listed = (await machine.sandbox.processes.list( - subStep(stepId, "reconcile"), + subStep(stepId, "reconcile", traceName.findStartedProcess), { limit: 250 }, )) as { items: Array<{ @@ -634,7 +718,7 @@ const pollUntilTerminal = async (opts: { const interval = pollIntervalsMs[index - 1] ?? maxPollIntervalMs; await opts.scope.run.step.sleep( - subStep(opts.stepId, `wait #${index}`), + subStep(opts.stepId, `wait #${index}`, traceName.wait(interval)), interval, ); @@ -642,7 +726,7 @@ const pollUntilTerminal = async (opts: { current = (await opts.machine.sandbox.processes.get( - subStep(opts.stepId, `check #${index}`), + subStep(opts.stepId, `check #${index}`, traceName.pollProcess), current.id, )) ?? current; } @@ -656,9 +740,12 @@ const readResult = async (opts: { argv: string[]; secrets: string[]; }): Promise => { - const output = await opts.process.getOutput(subStep(opts.stepId, "output"), { - tailBytes: outputTailBytes, - }); + const output = await opts.process.getOutput( + subStep(opts.stepId, "output", traceName.readOutput), + { + tailBytes: outputTailBytes, + }, + ); const decoded = decodeChunks(output); const stdout = tail(decoded.stdout, outputTailBytes); @@ -756,11 +843,12 @@ const decodeChunks = (output: { export const createRawCommand = ( getScope: () => CiJobScope, argv: string[], - label?: string, + /** How the command reads, when not as its arguments. */ + display: Pick = {}, ): Command => { return new CommandBuilder(getScope, { argv, - ...(label === undefined ? {} : { label }), + ...display, env: {}, retries: 0, nothrow: false, @@ -780,11 +868,9 @@ export const createCommandTag = ( tag.sh = (strings: TemplateStringsArray, ...values: CommandValue[]) => { const rendered = buildShellString([...strings], values); - return createRawCommand( - getScope, - ["/bin/sh", "-c", rendered], - truncateLabel(rendered), - ); + return createRawCommand(getScope, ["/bin/sh", "-c", rendered], { + text: truncateLabel(rendered), + }); }; return tag; diff --git a/packages/ci/src/machine/from.ts b/packages/ci/src/machine/from.ts index c2aa06dc4..7c1a7ed03 100644 --- a/packages/ci/src/machine/from.ts +++ b/packages/ci/src/machine/from.ts @@ -23,7 +23,12 @@ import { CiUsageError } from "../errors.ts"; import type { CacheBuildResult } from "../pipeline/cacheBuild.ts"; import { adoptBuilt, invokeBuild, validateInput } from "../pipeline/job.ts"; import type { CiJobScope, CiRunScope } from "../pipeline/scope.ts"; -import { countApi, rebuildSuffix, scopeSeparator } from "../pipeline/scope.ts"; +import { + countApi, + outsideJobs, + rebuildSuffix, + scopeSeparator, +} from "../pipeline/scope.ts"; import type { AnyJob, JobConfig, JobRef } from "../types.ts"; import { errorMessage, hash, stableStringify } from "../util.ts"; @@ -329,7 +334,7 @@ const buildOf = ( return existing; } - const built = (async () => { + const built = outsideJobs(run, async () => { const base = await baseOf(run, parent); const identity = base @@ -359,7 +364,7 @@ const buildOf = ( adoptBuilt(run, result); return result; - })(); + }); run.builds.set(path, built); @@ -472,7 +477,7 @@ const requestRebuild = ({ return existing; } - const built = (async () => { + const built = outsideJobs(run, async () => { let unnamed = replacing.unnamed; // Here, not in each child, so a snapshot that every child found broken is @@ -513,7 +518,7 @@ const requestRebuild = ({ adoptBuilt(run, result); return result; - })(); + }); run.builds.set(path, built); diff --git a/packages/ci/src/machine/machine.ts b/packages/ci/src/machine/machine.ts index ea328421e..a0c4f6779 100644 --- a/packages/ci/src/machine/machine.ts +++ b/packages/ci/src/machine/machine.ts @@ -9,6 +9,7 @@ import type { Inngest } from "inngest"; import type { CachedSnapshot, CacheTarget } from "../cache/cache.ts"; import { resolveTakenName, snapshotState } from "../cache/cache.ts"; import { CiUsageError } from "../errors.ts"; +import { ciSpan, ciStep, traceName } from "../pipeline/names.ts"; import type { CiJobScope, CiRunScope, @@ -16,10 +17,12 @@ import type { } from "../pipeline/scope.ts"; import { defaultCwd, + inJobSpan, isRestoring, runInScope, scopeSeparator, } from "../pipeline/scope.ts"; +import { inSpan } from "../pipeline/spans.ts"; import type { MachineConfig } from "../types.ts"; import { boundedName, @@ -54,12 +57,25 @@ export const machineName = (runId: string, path: string): string => { * Create this scope's machine if it doesn't have one yet. * * Machines are lazy: a job that never runs a command never gets one, and - * concurrent first commands share a single creation promise. + * concurrent first commands share a single creation promise. Its steps are in + * a span of its own, in its job's span rather than the first command's. */ export const ensureMachine = async ( scope: CiJobScope, ): Promise => { - scope.machine ??= createMachine(scope); + scope.machine ??= inJobSpan(scope.run, scope.jobPath, () => { + return inMachineSpan(scope, () => { + const stepId = `${scope.path}${scopeSeparator}machine`; + + // Everything in it is CI's work, including falling back from a + // snapshot that won't start. + const span = ciSpan(stepId, traceName.startMachine(scope.fromJobIds[0])); + + return inSpan(span, () => { + return createMachine(scope, stepId); + }); + }); + }); const handle = await scope.machine; @@ -84,6 +100,23 @@ export const ensureMachine = async ( return handle; }; +/** + * Run `fn` in an extra machine's span, so its commands sit apart from the + * job's own. A job's own machine has none: the job's span is its span. + */ +export const inMachineSpan = (scope: CiJobScope, fn: () => R): R => { + if (scope.path === scope.jobPath) { + return fn(); + } + + const span = { + id: scope.path, + name: traceName.extraMachine(scope.path, scope.jobPath), + }; + + return inSpan(span, fn); +}; + /** * Run once on every machine before its first command. * @@ -104,7 +137,11 @@ interface Started { sandbox: any; } -const createMachine = async (scope: CiJobScope): Promise => { +const createMachine = async ( + scope: CiJobScope, + /** The create step's ID, which other steps for the machine build on. */ + stepId: string, +): Promise => { const { run } = scope; const tools = run.sandboxTools; @@ -115,7 +152,7 @@ const createMachine = async (scope: CiJobScope): Promise => { } const name = machineName(run.runId, scope.path); - const stepId = `${scope.path}${scopeSeparator}machine`; + const create = { id: stepId, name: traceName.createMachine }; const machineConfig = resolveMachineConfig( scope.config.machine ?? run.machine ?? run.ci.defaultMachine, @@ -123,27 +160,29 @@ const createMachine = async (scope: CiJobScope): Promise => { /** Create a machine, from a snapshot if one is given, and set it up. */ const start = async ( - createStepId: string, + createStep: { id: string; name: string }, options: { name: string; snapshotId?: string; }, ): Promise => { const sandbox = options.snapshotId - ? await tools.create(createStepId, { + ? await tools.create(createStep, { name: options.name, snapshotId: options.snapshotId, }) - : await tools.create(createStepId, { + : await tools.create(createStep, { name: options.name, ...machineConfig, }); - await sandbox.commands.run(`${createStepId}${scopeSeparator}setup`, [ - "/bin/sh", - "-c", - machineSetupScript, - ]); + await sandbox.commands.run( + { + id: `${createStep.id}${scopeSeparator}setup`, + name: traceName.prepareWorkspace, + }, + ["/bin/sh", "-c", machineSetupScript], + ); return { sandbox }; }; @@ -155,9 +194,13 @@ const createMachine = async (scope: CiJobScope): Promise => { run.ci.hooks.activity(run, scope.jobPath, note); - return start(`${stepId}${scopeSeparator}fresh`, { - name: machineName(run.runId, `${scope.path} fresh`), - }); + return start( + { + id: `${stepId}${scopeSeparator}fresh`, + name: traceName.createFreshMachine, + }, + { name: machineName(run.runId, `${scope.path} fresh`) }, + ); }; const parentId = scope.fromJobIds[0] ?? "the parent"; @@ -166,6 +209,9 @@ const createMachine = async (scope: CiJobScope): Promise => { let started: Started; if (snapshotId) { + // Every job starting from a snapshot checks it for itself, so what a job + // does never depends on whether a sibling got there first. Only the + // rebuild of the parent, which each of them awaits, is shared. run.ci.hooks.activity( run, scope.jobPath, @@ -181,7 +227,7 @@ const createMachine = async (scope: CiJobScope): Promise => { let rebuild = { broken: false, unnamed: false }; try { - probed = await start(stepId, { name, snapshotId }); + probed = await start(create, { name, snapshotId }); } catch (error) { if (!isStartFailure(error)) { throw error; @@ -223,10 +269,13 @@ const createMachine = async (scope: CiJobScope): Promise => { // With the default wait, the same as the first try's: a shorter one // would make a slow node more likely to fail the retry, and a second // failure deletes the snapshot. - probed = await start(restartStepId, { - name: restartName, - snapshotId, - }); + probed = await start( + { id: restartStepId, name: traceName.restartMachine }, + { + name: restartName, + snapshotId, + }, + ); bad = undefined; @@ -273,10 +322,13 @@ const createMachine = async (scope: CiJobScope): Promise => { const retryName = machineName(run.runId, `${scope.path} retry`); try { - started = await start(retryStepId, { - name: retryName, - snapshotId: replacement, - }); + started = await start( + { id: retryStepId, name: traceName.retryCreate }, + { + name: retryName, + snapshotId: replacement, + }, + ); } catch (error) { if (!isStartFailure(error)) { throw error; @@ -299,7 +351,7 @@ const createMachine = async (scope: CiJobScope): Promise => { scope.startNote ?? "creating machine…", ); - started = await start(stepId, { name }); + started = await start(create, { name }); } const handle: MachineHandle = { @@ -313,7 +365,8 @@ const createMachine = async (scope: CiJobScope): Promise => { /** * Best-effort cleanup of a machine that failed to start. The platform keeps - * such a machine (stuck in STARTING) and its name, so it is destroyed here. + * such a machine (stuck in STARTING) and its name, so it is destroyed here, and + * the end of the run finds it by its name if this couldn't. * * Never throws: a machine that can't be cleaned up now must not fail the job. */ @@ -330,7 +383,7 @@ const discardFailedStart = async ( await run.step.run( { id: `${stepId}${scopeSeparator}discard`, - name: `${stepId}${scopeSeparator}discard`, + name: traceName.discardMachine, }, async (): Promise<{ id?: string }> => { // Errors are swallowed inside the step so it never retries. @@ -364,7 +417,7 @@ const discardFailedStart = async ( await sandbox?.destroy(); } } catch { - // Left to expire with the rest of the run's machines. + // The run's cleanup finds it again by its name. } return id ? { id } : {}; @@ -433,6 +486,14 @@ const errorStatus = (error: unknown): number | undefined => { return seen?.status ?? seen?.cause?.status; }; +/** Run `fn` in a job's "Save sandbox" span, which holds snapshotting it. */ +const inSaveSpan = (jobPath: string, fn: () => R): R => { + return inSpan( + ciSpan(`${jobPath}${scopeSeparator}save`, traceName.saveMachine), + fn, + ); +}; + /** How a job's snapshot is named. */ export interface SnapshotCache { target: CacheTarget; @@ -462,6 +523,11 @@ export interface TakenSnapshot { reused: boolean; } +/** A snapshot step: its ID is the step's, its name reads as an action. */ +const snapshotStep = (id: string) => { + return { id, name: traceName.snapshotMachine }; +}; + /** * Snapshot a job's own machine, which is the end of a build or a failed job * that keeps its machine. With a cache, the snapshot is named after the job's @@ -483,7 +549,9 @@ export const snapshotMachine = async ( const handle = await scope.machine; - return takeSnapshot(run, scope.path, handle, cache); + return inSaveSpan(scope.path, () => { + return takeSnapshot(run, scope.path, handle, cache); + }); }; const takeSnapshot = async ( @@ -533,7 +601,7 @@ const createNamedSnapshot = async ( const name = target.name; const attempt = async (id: string): Promise => { - const snapshot = await handle.sandbox.snapshot(id, { name }); + const snapshot = await handle.sandbox.snapshot(snapshotStep(id), { name }); return { snapshotId: snapshot.id, @@ -590,7 +658,9 @@ const createNamedSnapshot = async ( ); } - const unnamed = await handle.sandbox.snapshot(`${stepId} (unnamed)`); + const unnamed = await handle.sandbox.snapshot( + snapshotStep(`${stepId} (unnamed)`), + ); // The name was refused or couldn't be freed, so the job's snapshot falls // back to an unnamed one that no later run can find. The run that invoked @@ -608,7 +678,7 @@ const createRunSnapshot = async ( handle: MachineHandle, stepId: string, ): Promise => { - const snapshot = await handle.sandbox.snapshot(stepId); + const snapshot = await handle.sandbox.snapshot(snapshotStep(stepId)); return { snapshotId: snapshot.id, reused: false }; }; @@ -689,10 +759,10 @@ export const destroyRunMachines = async ( attempt = 0, ): Promise => { await run.step.run( - { - id: `pipeline${scopeSeparator}cleanup${attempt > 0 ? ` (attempt ${attempt})` : ""}`, - name: "cleanup", - }, + ciStep( + `pipeline${scopeSeparator}cleanup${attempt > 0 ? ` (attempt ${attempt})` : ""}`, + traceName.cleanUpMachines, + ), async () => { return destroyOrphans(run.ci.client, run.runId); }, @@ -722,10 +792,10 @@ export const deleteRunSnapshots = async ( // Always there, and reads the set when it runs, for the reason cleaning up // machines is. await run.step.run( - { - id: `pipeline${scopeSeparator}cleanup:snapshots${attempt > 0 ? ` (attempt ${attempt})` : ""}`, - name: "cleanup:snapshots", - }, + ciStep( + `pipeline${scopeSeparator}cleanup:snapshots${attempt > 0 ? ` (attempt ${attempt})` : ""}`, + traceName.cleanUpSnapshots, + ), async () => { const deleted: string[] = []; const failed: string[] = []; diff --git a/packages/ci/src/machine/spans.test.ts b/packages/ci/src/machine/spans.test.ts new file mode 100644 index 000000000..681b0df14 --- /dev/null +++ b/packages/ci/src/machine/spans.test.ts @@ -0,0 +1,583 @@ +/** + * Tests of the trace spans CI groups its steps under, and what it names them: + * one span per job, and in it one per machine, one per command, and one per + * attempt for a command with retries, with GitHub check updates in a span of + * their own. Spans and names never change a step's ID. A failed run of a + * command ends in a failing step, so its span shows the failure. CI's own + * work carries CI's origin, and what you wrote carries none. + * + * @module + */ + +import { version as sdkVersion, step } from "inngest"; +import { describe, expect, test } from "vitest"; +import { CommandFailedError } from "../errors.ts"; +import { consoleReporter } from "../github/auth.ts"; +import { createCi } from "../pipeline/createCi.ts"; +import { ciOrigin } from "../pipeline/names.ts"; +import { createCiTestClient } from "../testing/client.ts"; +import { prEvent } from "../testing/events.ts"; +import type { CommandScript, FakeSandboxApi } from "../testing/fakeSandbox.ts"; +import { createFakeSandboxApi } from "../testing/fakeSandbox.ts"; +import { runFunction } from "../testing/runFunction.ts"; +import { version } from "../version.ts"; +import { $ } from "./command.ts"; +import { sandbox } from "./sandbox.ts"; + +type Ci = ReturnType; + +interface TraceNode { + label: string; + children: Map; +} + +/** + * Draw a run's trace as an indented tree, the way the UI nests it: each span + * once, where its first step is, with its kind in brackets, and each step + * under its spans. + */ +const drawTrace = ( + result: Awaited>, + /** Suffix each row with who it says did it, as in `Create sandbox <- ci`. */ + withOrigins = false, +): string => { + const root: TraceNode = { label: "", children: new Map() }; + + const by = (origin: string | undefined) => { + if (!withOrigins || !origin) { + return ""; + } + + const names: Record = { + [ciOrigin]: "ci", + [`inngest@${sdkVersion}`]: "inngest", + }; + + return ` <- ${names[origin] ?? origin}`; + }; + + for (const stepId of result.stepIds) { + let node = root; + + for (const span of result.spans[stepId] ?? []) { + const key = `span:${span.id}`; + const label = span.kind ? `${span.name} [${span.kind}]` : span.name; + + const child = node.children.get(key) ?? { + label: `${label}${by(span.origin)}`, + children: new Map(), + }; + + node.children.set(key, child); + + node = child; + } + + node.children.set(`step:${stepId}`, { + label: `${result.names[stepId] ?? stepId}${by(result.origins[stepId])}`, + children: new Map(), + }); + } + + const lines: string[] = []; + + const draw = (node: TraceNode, depth: number) => { + for (const child of node.children.values()) { + lines.push(`${" ".repeat(depth)}${child.label}`); + + draw(child, depth + 1); + } + }; + + draw(root, 0); + + return lines.join("\n"); +}; + +/** Run the pipeline whose handler `define` makes from a fresh client. */ +const runPipeline = async ( + define: (ci: Ci) => () => Promise, + scripts: CommandScript[] = [], + api: FakeSandboxApi = createFakeSandboxApi(), +) => { + api.script(scripts); + + const ci = createCi(createCiTestClient(api), { + github: consoleReporter(), + }); + + const pipeline = ci.pipeline( + { id: "pr", on: [{ event: "github/pull_request.opened" }] }, + define(ci), + ); + + return runFunction(pipeline, { event: prEvent }); +}; + +/** Run a job with a captured, a managed, a retried and a background command. */ +const run = () => { + return runPipeline( + (ci) => { + return ci.job("test", async () => { + await $`quick`.timeout("30s"); + + await $`slow`; + + try { + await $`flaky`.retries(1); + } catch { + // Both attempts fail. + } + + const server = await $`serve`.background(); + + await server.output(); + + await server.exited(); + + await server.kill(); + }); + }, + [ + { match: "slow", ticks: 1, stdout: "done" }, + { match: "flaky", exitCode: 1 }, + { match: "serve", ticks: 1, stdout: "listening" }, + ], + ); +}; + +/** + * A pipeline with a row of every sort: a job started from another, plain and + * labelled commands, a step of your own and an extra sandbox. + */ +const rowsPipeline = (ci: Ci) => { + const base = ci.job( + { id: "base", name: "Install dependencies" }, + async () => { + await $`pnpm install`; + }, + ); + + return ci.job({ id: "test", from: base }, async () => { + await $`pnpm test`; + + await $`pnpm exec eslint .`.as("lint"); + + await step.run("my-step", () => { + return "done"; + }); + + const api = await sandbox("api"); + + await api.$`pnpm start`.timeout("30s"); + }); +}; + +/** The span of a command `test` ran. */ +const command = (label: string) => { + return { id: `test › ${label}`, name: `$ ${label}` }; +}; + +const origin = ciOrigin; +const github = [{ id: "github", name: "GitHub", origin }]; +const job = { id: "test", name: "test", kind: "job" }; +const machine = [job, { id: "test › machine", name: "Start sandbox", origin }]; +const quick = [job, command("quick")]; +const slow = [job, command("slow")]; +const attempt1 = [ + job, + command("flaky"), + { id: "attempt-1", name: "Attempt 1", origin }, +]; +const attempt2 = [ + job, + command("flaky"), + { id: "attempt-2", name: "Attempt 2", origin }, +]; +const serve = [job, command("serve")]; + +describe("spans", () => { + test("group each job, machine, command and retried attempt", async () => { + const result = await run(); + + expect(result.type).toBe("function-resolved"); + + // Background calls re-enter their command's span. Check updates are in + // the run's GitHub span, and cleanup is in none. + expect(result.spans).toEqual({ + "github › check:pr:start": github, + "github › check:test:start": github, + "test › machine": machine, + "test › machine › setup": machine, + "test › quick": quick, + "test › slow › start": slow, + "test › slow › wait #1": slow, + "test › slow › check #1": slow, + "test › slow › wait #2": slow, + "test › slow › check #2": slow, + "test › slow › output": slow, + "test › flaky #attempt-1 › start": attempt1, + "test › flaky #attempt-1 › wait #1": attempt1, + "test › flaky #attempt-1 › check #1": attempt1, + "test › flaky #attempt-1 › output": attempt1, + "test › flaky #attempt-1 › exit": attempt1, + "github › check:test:attempt:1": github, + "test › flaky #attempt-2 › start": attempt2, + "test › flaky #attempt-2 › wait #1": attempt2, + "test › flaky #attempt-2 › check #1": attempt2, + "test › flaky #attempt-2 › output": attempt2, + "test › flaky #attempt-2 › exit": attempt2, + "test › serve › start": serve, + "test › serve › output #1": serve, + "test › serve › wait #2": serve, + "test › serve › check #2": serve, + "test › serve › wait #3": serve, + "test › serve › check #3": serve, + "test › serve › output": serve, + "test › serve › kill": serve, + "github › check:test:complete": github, + "github › check:jobs:complete": github, + "github › check:pr:complete": github, + }); + }); + + test("leave step IDs and their order as they were, plus one per failed run", async () => { + const result = await run(); + + expect(result.stepIds).toEqual([ + "github › check:pr:start", + "github › check:test:start", + "test › machine", + "test › machine › setup", + "test › quick", + "test › slow › start", + "test › slow › wait #1", + "test › slow › check #1", + "test › slow › wait #2", + "test › slow › check #2", + "test › slow › output", + "test › flaky #attempt-1 › start", + "test › flaky #attempt-1 › wait #1", + "test › flaky #attempt-1 › check #1", + "test › flaky #attempt-1 › output", + "test › flaky #attempt-1 › exit", + "github › check:test:attempt:1", + "test › flaky #attempt-2 › start", + "test › flaky #attempt-2 › wait #1", + "test › flaky #attempt-2 › check #1", + "test › flaky #attempt-2 › output", + "test › flaky #attempt-2 › exit", + "test › serve › start", + "test › serve › output #1", + "test › serve › wait #2", + "test › serve › check #2", + "test › serve › wait #3", + "test › serve › check #3", + "test › serve › output", + "test › serve › kill", + "github › check:test:complete", + "github › check:jobs:complete", + "github › check:pr:complete", + "pipeline › cleanup", + "pipeline › cleanup:snapshots", + ]); + }); + + test("name each step for what it does, not for what its spans show", async () => { + const result = await run(); + + expect(result.names).toMatchObject({ + "github › check:pr:start": "Create check: pr", + "github › check:test:start": "Report test: started", + "test › machine": "Create sandbox", + "test › machine › setup": "Prepare workspace", + "test › quick": "Run and read output", + "test › slow › start": "Start process", + "test › slow › wait #1": "Wait 1s", + "test › slow › check #1": "Poll process", + "test › slow › wait #2": "Wait 2s", + "test › slow › output": "Read output", + "github › check:test:attempt:1": "Report test: retrying (attempt 2 of 2)", + "test › serve › output #1": "Read output", + "test › serve › kill": "Stop process", + "github › check:test:complete": "Report test: passed", + "github › check:jobs:complete": "Report jobs: ended with the run", + "github › check:pr:complete": "Complete check: pr", + "pipeline › cleanup": "Clean up sandboxes", + }); + }); + + test("end a failed run's span in a failing step named for its exit", async () => { + const result = await run(); + + const attempt1Steps = result.stepIds.filter((id) => { + return id.startsWith("test › flaky #attempt-1"); + }); + + expect(attempt1Steps.at(-1)).toBe("test › flaky #attempt-1 › exit"); + + expect(result.spans["test › flaky #attempt-1 › exit"]).toEqual(attempt1); + + expect(result.names["test › flaky #attempt-1 › exit"]).toBe( + "Exited with code 1", + ); + + expect(result.steps["test › flaky #attempt-1 › exit"]).toMatchObject({ + name: "NonRetriableError", + message: "exit 1", + }); + }); + + test("record nothing more for a passing or `.nothrow()` command", async () => { + const result = await runPipeline( + (ci) => { + return ci.job("test", async () => { + await $`pass`; + + await $`fail`.nothrow(); + }); + }, + [{ match: "fail", exitCode: 1 }], + ); + + expect(result.type).toBe("function-resolved"); + + expect(result.stepIds).toEqual([ + "github › check:pr:start", + "github › check:test:start", + "test › machine", + "test › machine › setup", + "test › pass › start", + "test › pass › wait #1", + "test › pass › check #1", + "test › pass › output", + "test › fail › start", + "test › fail › wait #1", + "test › fail › check #1", + "test › fail › output", + "github › check:test:complete", + "github › check:jobs:complete", + "github › check:pr:complete", + "pipeline › cleanup", + "pipeline › cleanup:snapshots", + ]); + }); + + test("still retry after a failed run, and throw the same error at the end", async () => { + let caught: unknown; + + const result = await runPipeline( + (ci) => { + return ci.job("test", async () => { + try { + await $`flaky`.retries(1); + } catch (error) { + caught = error; + } + }); + }, + [{ match: "flaky", exitCode: 1, stdout: "out", stderr: "flake" }], + ); + + expect(result.type).toBe("function-resolved"); + + expect(result.stepIds).toContain("test › flaky #attempt-2 › exit"); + + expect(caught).toBeInstanceOf(CommandFailedError); + + expect(caught).toMatchObject({ + message: "`flaky` exited with 1\nflake", + command: ["flaky"], + exitCode: 1, + stdoutTail: "out", + stderrTail: "flake", + jobPath: "test", + }); + }); + + test("keep the build of a parent outside every job's span", async () => { + const result = await runPipeline((ci) => { + const base = ci.job("base", async () => { + await $`install`; + }); + + return ci.job({ id: "test", from: base }, async () => { + await $`unit`; + }); + }); + + expect(result.type).toBe("function-resolved"); + + const jobOf = Object.fromEntries( + Object.entries(result.spans).map(([stepId, spans]) => { + return [stepId, spans[0]?.id]; + }), + ); + + // `base` runs in a run of its own, so the pipeline has only the steps that + // look it up and invoke it, outside any job whichever child asks first. + // Check updates are in the GitHub span. + expect(result.stepIds).toContain("base (from) › lookup"); + expect(result.stepIds).toContain("base (from) › build"); + expect(jobOf).toEqual({ + "github › check:pr:start": "github", + "github › check:test:start": "github", + "test › machine": "test", + "test › machine › setup": "test", + "test › unit › start": "test", + "test › unit › wait #1": "test", + "test › unit › check #1": "test", + "test › unit › output": "test", + "github › check:test:complete": "github", + "github › check:jobs:complete": "github", + "github › check:pr:complete": "github", + }); + }); + + test("name a pipeline's rows for what they are", async () => { + const result = await runPipeline(rowsPipeline); + + expect(result.type).toBe("function-resolved"); + + expect(drawTrace(result)).toBe( + [ + "GitHub", + " Create check: pr", + " Report test: started", + " Report test: passed", + " Report jobs: ended with the run", + " Complete check: pr", + "Look up cache", + "Build base in its own run", + "test [job]", + " Start sandbox from base", + " Create sandbox", + " Prepare workspace", + " $ pnpm test", + " Start process", + " Wait 1s", + " Poll process", + " Read output", + " lint", + " Start process", + " Wait 1s", + " Poll process", + " Read output", + " my-step", + " api", + " Start sandbox", + " Create sandbox", + " Prepare workspace", + " $ pnpm start", + " Run and read output", + "Clean up sandboxes", + "Clean up snapshots", + ].join("\n"), + ); + }); + + test("mark the work CI does for you, and nothing you wrote", async () => { + const result = await runPipeline(rowsPipeline); + + expect(result.type).toBe("function-resolved"); + + expect(ciOrigin).toBe(`@inngest/ci@${version}`); + + // Jobs, commands, `my-step` and the `api` sandbox are yours. `snapshot()` + // opens its span without an origin, so it inherits the one of the CI span + // it runs in, while its own steps are marked by the SDK. + expect(drawTrace(result, true)).toBe( + [ + "GitHub <- ci", + " Create check: pr <- ci", + " Report test: started <- ci", + " Report test: passed <- ci", + " Report jobs: ended with the run <- ci", + " Complete check: pr <- ci", + "Look up cache <- ci", + "Build base in its own run <- ci", + "test [job]", + " Start sandbox from base <- ci", + " Create sandbox <- ci", + " Prepare workspace <- ci", + " $ pnpm test", + " Start process <- ci", + " Wait 1s <- ci", + " Poll process <- ci", + " Read output <- ci", + " lint", + " Start process <- ci", + " Wait 1s <- ci", + " Poll process <- ci", + " Read output <- ci", + " my-step", + " api", + " Start sandbox <- ci", + " Create sandbox <- ci", + " Prepare workspace <- ci", + " $ pnpm start", + " Run and read output <- ci", + "Clean up sandboxes <- ci", + "Clean up snapshots <- ci", + ].join("\n"), + ); + }); + + test("mark every step a command runs, through retries and background calls", async () => { + const result = await run(); + + expect(result.type).toBe("function-resolved"); + + // The job only runs commands, so every step in it is CI's. + for (const stepId of result.stepIds) { + expect(result.origins[stepId]).toBe(ciOrigin); + } + }); + + test("ask for a parent's snapshot again, as a build of its own, when it won't start", async () => { + const api = createFakeSandboxApi(); + const { fetch } = api; + let broken = false; + + // Break the snapshot as soon as it's taken, before `test` starts from it. + api.fetch = (input, init) => { + if (!broken && api.snapshots.size > 0) { + broken = true; + + api.failSnapshotStarts(); + } + + return fetch(input, init); + }; + + const result = await runPipeline( + (ci) => { + const parent = ci.job("base", async () => { + await $`pnpm install`; + }); + + return ci.job({ id: "test", from: parent }, async () => { + await $`pnpm test`; + }); + }, + [], + api, + ); + + expect(result.type).toBe("function-resolved"); + + const builds = result.stepIds.filter((id) => { + return id.endsWith("› build"); + }); + + expect(builds).toEqual([ + "base (from) › build", + "base (from) (rebuild) › build", + ]); + + // Neither is in a job's span. + for (const id of builds) { + expect(result.spans[id]?.[0]).toBeUndefined(); + } + }); +}); diff --git a/packages/ci/src/pipeline/README.md b/packages/ci/src/pipeline/README.md index 42650525b..d6748126e 100644 --- a/packages/ci/src/pipeline/README.md +++ b/packages/ci/src/pipeline/README.md @@ -10,5 +10,7 @@ The client and the run engine. - `matrix.ts`: `ci.matrix()`, matrix expansion and the concurrency pool. - `metadata.ts`: the `userland.inngest-ci` metadata attached to runs and steps, and the helpers that build and attach it. - `scope.ts`: the run and job scopes held in async context. Jobs only write to the run scope, except for `builds`, the promises of the builds `from` parents need. +- `names.ts`: what each step and span CI writes is called in the trace, and the origin that marks CI's own work. +- `spans.ts`: the one place CI touches the SDK's experimental trace-span API. It is feature-detected, so CI runs unchanged on an SDK without it. - `durable.ts`: durable proxies that run calls as steps. - `rerun.ts`: re-running a pipeline from a GitHub check. diff --git a/packages/ci/src/pipeline/cacheGuarantees.test.ts b/packages/ci/src/pipeline/cacheGuarantees.test.ts index d1b05bb45..c34aa0b88 100644 --- a/packages/ci/src/pipeline/cacheGuarantees.test.ts +++ b/packages/ci/src/pipeline/cacheGuarantees.test.ts @@ -471,6 +471,18 @@ describe("jobs that start from one parent", () => { expect(buildSteps(result.stepIds)).toEqual(["install (from) › build"]); expect(count(api, "pnpm install")).toBe(1); + + for (const job of ["a", "b"]) { + const inJob = result.stepIds.filter((stepId) => { + return result.spans[stepId]?.[0]?.id === job; + }); + + expect(inJob[0]).toBe(`${job} › machine`); + expect(result.spans[`${job} › machine`]?.[0]?.name).toBe(job); + } + + // Many jobs share the build, so it sits in none of them. + expect(result.spans["install (from) › build"] ?? []).toEqual([]); }); test("a warm hit invokes no build, and the parent is still looked up once", async () => { diff --git a/packages/ci/src/pipeline/durable.ts b/packages/ci/src/pipeline/durable.ts index b8971aab1..757ef95fa 100644 --- a/packages/ci/src/pipeline/durable.ts +++ b/packages/ci/src/pipeline/durable.ts @@ -178,7 +178,8 @@ const call = async ( // inside the handler; that's how the call knows which repository it's for. const step = getRunScope()?.step ?? execution.ctx.step; - return step.run({ id, name: overrides.name ?? id }, invoke); + // Named by the method called, without the job path or count in its ID. + return step.run({ id, name: overrides.name ?? label }, invoke); }; const invokeOnClient = async ( diff --git a/packages/ci/src/pipeline/job.ts b/packages/ci/src/pipeline/job.ts index 8528e9c71..447587df6 100644 --- a/packages/ci/src/pipeline/job.ts +++ b/packages/ci/src/pipeline/job.ts @@ -33,9 +33,11 @@ import type { AnyJob, CheckConclusion, JobConfig } from "../types.ts"; import { errorMessage, formatDuration, shortReason } from "../util.ts"; import type { CacheBuildData, CacheBuildResult } from "./cacheBuild.ts"; import { tagStep } from "./metadata.ts"; +import { ciStep, traceName } from "./names.ts"; import type { CiJobScope, CiRunScope } from "./scope.ts"; import { getRunScope, + inJobSpan, matrixOriginOf, rootRunIdOf, runJobBody, @@ -231,10 +233,11 @@ export const invokeBuild = async ({ let output: CacheBuildResult | null; try { - const buildStepId = `${stepPath}${scopeSeparator}build`; - output = (await run.step.invoke( - { id: buildStepId, name: buildStepId }, + ciStep( + `${stepPath}${scopeSeparator}build`, + traceName.buildInOwnRun(path), + ), { function: run.ci.cacheBuild(), data }, )) as CacheBuildResult | null; } catch (error) { @@ -394,7 +397,16 @@ export const validateInput = async ( ); }; -const jobBody = async ({ +/** Everything a job does is in its span. */ +const jobBody = ( + args: RunJobArgs & { run: CiRunScope; path: string; number: number }, +): Promise => { + return inJobSpan(args.run, args.path, () => { + return jobSteps(args); + }); +}; + +const jobSteps = async ({ run, config: declared, handler, @@ -536,7 +548,12 @@ const jobBody = async ({ const checkStartedAt = checked ? await checks.jobStart(target) : undefined; const startedAt = checkStartedAt ?? - (await durableNow(run, `start:${scope.path}`, scope.path)); + (await durableNow( + run, + `start:${scope.path}`, + traceName.recordStartTime, + scope.path, + )); if (checked) { run.openChecks.set(scope.path, checkName); @@ -614,7 +631,13 @@ const jobBody = async ({ } const endedAt = - checkEndedAt ?? (await durableNow(run, `end:${scope.path}`, scope.path)); + checkEndedAt ?? + (await durableNow( + run, + `end:${scope.path}`, + traceName.recordEndTime, + scope.path, + )); const durationMs = endedAt - startedAt; run.summaries.push({ @@ -658,7 +681,13 @@ const jobBody = async ({ } const endedAt = - checkEndedAt ?? (await durableNow(run, `end:${scope.path}`, scope.path)); + checkEndedAt ?? + (await durableNow( + run, + `end:${scope.path}`, + traceName.recordEndTime, + scope.path, + )); run.summaries.push({ path: scope.path, @@ -679,9 +708,10 @@ const jobBody = async ({ const durableNow = ( run: CiRunScope, id: string, + name: string, jobPath: string, ): Promise => { - return run.step.run({ id, name: id }, async () => { + return run.step.run(ciStep(id, name), async () => { await tagStep(run, { kind: "job", job: jobPath }); return Date.now(); diff --git a/packages/ci/src/pipeline/metadata.test.ts b/packages/ci/src/pipeline/metadata.test.ts index 3984c8ee8..a9e6571e2 100644 --- a/packages/ci/src/pipeline/metadata.test.ts +++ b/packages/ci/src/pipeline/metadata.test.ts @@ -19,6 +19,7 @@ import { runFunction } from "../testing/runFunction.ts"; import { version } from "../version.ts"; import { createCi } from "./createCi.ts"; import { tagStep } from "./metadata.ts"; +import { ciOrigin } from "./names.ts"; import type { CiRunScope } from "./scope.ts"; const setup = () => { @@ -250,6 +251,9 @@ describe("run metadata", () => { expect(result.stepIds).toContain("ci › metadata:start"); expect(result.stepIds).toContain("ci › metadata:end"); + + expect(result.origins["ci › metadata:start"]).toBe(ciOrigin); + expect(result.origins["ci › metadata:end"]).toBe(ciOrigin); }); test("a pipeline with its check on adds no steps of its own", async () => { @@ -306,6 +310,11 @@ describe("step metadata", () => { expect(update.kind).toBe("userland.inngest-ci"); expect(update.op).toBe("merge"); } + + // Caching, start times and checks are all CI's work, not the jobs'. + for (const [step] of tags) { + expect(result.origins[step as string]).toBe(ciOrigin); + } }); }); diff --git a/packages/ci/src/pipeline/metadata.ts b/packages/ci/src/pipeline/metadata.ts index ce6a8986d..72f2c1657 100644 --- a/packages/ci/src/pipeline/metadata.ts +++ b/packages/ci/src/pipeline/metadata.ts @@ -11,6 +11,7 @@ import { getAsyncCtx } from "inngest/experimental"; import type { CheckConclusion } from "../types.ts"; import { version } from "../version.ts"; +import { ciStep, traceName } from "./names.ts"; import type { CiRunScope } from "./scope.ts"; import { apiNames } from "./scope.ts"; @@ -149,7 +150,7 @@ export const metadataStep = ( id: string, runValues: () => Record, ): Promise => { - return run.step.run({ id, name: id }, async () => { + return run.step.run(ciStep(id, traceName.recordRunDetails), async () => { await tagStep(run, undefined, runValues()); return Date.now(); diff --git a/packages/ci/src/pipeline/names.ts b/packages/ci/src/pipeline/names.ts new file mode 100644 index 000000000..82964bf51 --- /dev/null +++ b/packages/ci/src/pipeline/names.ts @@ -0,0 +1,160 @@ +/** + * What every step and span CI writes is called in the trace. Rows you wrote + * read the way you wrote them, and CI's own work reads as a plain-English + * action. Names are for reading only: step IDs, which replays depend on, are + * built where each step is and never from a name. + * + * CI's own work is also marked with CI's origin, so a trace can set it apart + * from what you wrote. Rows you wrote are never marked. + * + * @module + */ + +import type { CheckConclusion } from "../types.ts"; +import { formatDuration } from "../util.ts"; +import { version } from "../version.ts"; +import type { CiRunScope } from "./scope.ts"; +import { scopeSeparator } from "./scope.ts"; +import { originOption } from "./spans.ts"; + +/** + * Marks a step or span as work CI does for you rather than something you + * wrote, as the SDK's experimental step origin and span `origin`. + * Steps inherit it from the innermost span that has one, so marking a span + * marks everything in it. + */ +export const ciOrigin = `@inngest/ci@${version}`; + +/** A span CI opens for its own work, such as starting a sandbox. */ +export const ciSpan = (id: string, name: string) => { + return { id, name, origin: ciOrigin }; +}; + +/** + * A step CI runs for you where no span of CI's marks it, such as a command's + * `Start process` inside the command's span, which is yours. + */ +export const ciStep = (id: string, name: string) => { + return { id, name, ...originOption(ciOrigin) }; +}; + +/** How a finished check reads, as in `Report test: passed`. */ +const outcomes: Partial> = { + success: "passed", + failure: "failed", + timed_out: "timed out", +}; + +export const traceName = { + /** A job's span: its `name`, or its path when it has none. */ + job: (run: CiRunScope, path: string): string => { + return run.ci.jobs.get(path)?.config.name ?? path; + }, + + /** The span a job's sandbox starts in. */ + startMachine: (parent?: string): string => { + return parent ? `Start sandbox from ${parent}` : "Start sandbox"; + }, + + /** The span of an extra sandbox, named as `sandbox()` was. */ + extraMachine: (path: string, jobPath: string): string => { + return path.slice(jobPath.length + scopeSeparator.length); + }, + + /** A command's span: its `.as()` label, or the command itself. */ + command: (text: string, label?: string): string => { + return label ?? `$ ${text}`; + }, + + attempt: (attempt: number): string => { + return `Attempt ${attempt}`; + }, + + wait: (ms: number): string => { + return `Wait ${formatDuration(ms)}`; + }, + + exited: (exitCode: number): string => { + return `Exited with code ${exitCode}`; + }, + + buildInOwnRun: (path: string): string => { + return `Build ${path} in its own run`; + }, + + createCheck: (check: string): string => { + return `Create check: ${check}`; + }, + + completeCheck: (check: string): string => { + return `Complete check: ${check}`; + }, + + /** A check update, as in `Report test: started`. */ + report: (subject: string, status: string): string => { + return `Report ${subject}: ${status}`; + }, + + retrying: (attempt: number, of: number): string => { + return `retrying (attempt ${attempt} of ${of})`; + }, + + outcome: (conclusion: CheckConclusion): string => { + return outcomes[conclusion] ?? conclusion; + }, + + /** A `github.*` helper you called, by its method name. */ + githubHelper: (helper: string): string => { + return `github.${helper}`; + }, + + canUser: (login: string, permission: string): string => { + return `Check ${login} can ${permission}`; + }, + + waitForCheck: (name: string): string => { + return `Wait for check: ${name}`; + }, + + waitForWorkflow: (workflow: string): string => { + return `Wait for workflow: ${workflow}`; + }, + + github: "GitHub", + resolveRepository: "Resolve repository", + commentNotAllowed: "Comment: not allowed", + + createMachine: "Create sandbox", + restartMachine: "Restart sandbox", + createFreshMachine: "Create sandbox (fresh)", + retryCreate: "Retry create", + discardMachine: "Discard sandbox", + prepareWorkspace: "Prepare workspace", + saveMachine: "Save sandbox", + snapshotMachine: "Snapshot sandbox", + cleanUpMachines: "Clean up sandboxes", + cleanUpSnapshots: "Clean up snapshots", + + /** A captured command's one step: it runs and returns its output at once. */ + runAndReadOutput: "Run and read output", + startProcess: "Start process", + pollProcess: "Poll process", + readOutput: "Read output", + stopProcess: "Stop process", + stopAfterTimeout: "Stop after timeout", + findStartedProcess: "Find started process", + + checkCache: "Check cache", + lookUpCache: "Look up cache", + resolveCacheName: "Resolve cache name", + deleteBadSnapshot: "Delete bad snapshot", + checkSnapshotState: "Check snapshot state", + + findChangedFiles: "Find changed files", + + recordStartTime: "Record start time", + recordEndTime: "Record end time", + recordRunDetails: "Record run details", + addSummary: "Add summary", + addAnnotations: "Add annotations", +}; diff --git a/packages/ci/src/pipeline/pipeline.ts b/packages/ci/src/pipeline/pipeline.ts index 1c3bd4c25..4b5c9ce63 100644 --- a/packages/ci/src/pipeline/pipeline.ts +++ b/packages/ci/src/pipeline/pipeline.ts @@ -49,10 +49,12 @@ import type { CacheBuildData } from "./cacheBuild.ts"; import type { RegisteredJob } from "./job.ts"; import { conclusionForError, runJob } from "./job.ts"; import { metadataStep, runEndMetadata, runStartMetadata } from "./metadata.ts"; +import { ciStep, traceName } from "./names.ts"; import type { CiInternals, CiRunScope } from "./scope.ts"; import { apiNames, countApi, + inGitHubSpan, initCiAls, runInScope, withScopePreserved, @@ -566,38 +568,41 @@ const resolveConfiguredRepo = ( ): Promise => { const { owner, name } = parseRepo(fullName); - return run.step.run( - { id: `github › repo:resolve`, name: "repo:resolve" }, - async (): Promise => { - const base: RepoContext = { owner, name, fullName, sha: "" }; - const provider = run.ci.github as GitHubProvider; - - if (provider.kind === "console") { - return base; - } - - const octokit = await provider.octokit({ owner, repo: name }); - const { data: info } = await octokit.rest.repos.get({ - owner, - repo: name, - }); - const branch = info.default_branch; + return inGitHubSpan(run, () => { + return run.step.run( + { id: `github › repo:resolve`, name: traceName.resolveRepository }, + async (): Promise => { + const base: RepoContext = { owner, name, fullName, sha: "" }; + const provider = run.ci.github as GitHubProvider; + + if (provider.kind === "console") { + return base; + } + + const octokit = await provider.octokit({ owner, repo: name }); + const { data: info } = await octokit.rest.repos.get({ + owner, + repo: name, + }); + const branch = info.default_branch; - const { data: head } = await octokit.rest.repos.getBranch({ - owner, - repo: name, - branch, - }); + const { data: head } = await octokit.rest.repos.getBranch({ + owner, + repo: name, + branch, + }); - return { - ...base, - sha: head.commit.sha, - ref: `refs/heads/${branch}`, - baseRef: branch, - trigger: (run.event as { name?: string } | undefined)?.name ?? "manual", - }; - }, - ) as Promise; + return { + ...base, + sha: head.commit.sha, + ref: `refs/heads/${branch}`, + baseRef: branch, + trigger: + (run.event as { name?: string } | undefined)?.name ?? "manual", + }; + }, + ) as Promise; + }); }; /** @@ -826,19 +831,21 @@ const checkCommentPermission = async ( const allowed = await canUser(login, minPermission); if (!allowed) { - await run.step.run( - { id: "github › comment:denied", name: "comment:denied" }, - async () => { - const { stickyComment } = await import("../github/helpers.ts"); - - await stickyComment( - "permission", - `@${login} you need \`${minPermission}\` permission to run \`${commentBody(run.event as { data?: unknown }).split(" ")[0]}\`.`, - ); - - return { denied: login }; - }, - ); + await inGitHubSpan(run, () => { + return run.step.run( + { id: "github › comment:denied", name: traceName.commentNotAllowed }, + async () => { + const { stickyComment } = await import("../github/helpers.ts"); + + await stickyComment( + "permission", + `@${login} you need \`${minPermission}\` permission to run \`${commentBody(run.event as { data?: unknown }).split(" ")[0]}\`.`, + ); + + return { denied: login }; + }, + ); + }); } return allowed; @@ -892,9 +899,12 @@ export const cleanupFunction = ({ async ({ event, step }: any) => { const runId = event?.data?.run_id ?? event?.data?.runId; - return step.run("destroy-orphans", async () => { - return runId ? destroyOrphans(client, runId) : { destroyed: 0 }; - }); + return step.run( + ciStep("destroy-orphans", traceName.cleanUpMachines), + async () => { + return runId ? destroyOrphans(client, runId) : { destroyed: 0 }; + }, + ); }, ); }; diff --git a/packages/ci/src/pipeline/scope.ts b/packages/ci/src/pipeline/scope.ts index 36ee0c732..653f1c2ba 100644 --- a/packages/ci/src/pipeline/scope.ts +++ b/packages/ci/src/pipeline/scope.ts @@ -7,7 +7,7 @@ import type { GetStepTools, Inngest, InngestFunction } from "inngest"; import type { AsyncContext, DurableSandboxTools } from "inngest/experimental"; -import { runWithAsyncCtx } from "inngest/experimental"; +import { getAsyncCtx, runWithAsyncCtx } from "inngest/experimental"; import type { CachedSnapshot } from "../cache/cache.ts"; import { CiUsageError } from "../errors.ts"; import type { @@ -19,6 +19,8 @@ import type { } from "../types.ts"; import type { CacheBuildData, CacheBuildResult } from "./cacheBuild.ts"; import type { CiHooks } from "./hooks.ts"; +import { ciSpan, traceName } from "./names.ts"; +import { inSpan } from "./spans.ts"; /** * The separator used between parts of a scope path and a step label. It's a @@ -510,9 +512,16 @@ export const withScopePreserved = (tools: T): T => { return withStepIdPrefix(tools); }; +/** + * Prefix a step's ID with its scope path, and name it by the ID it was given, + * since its job's span already shows the rest. + */ const prefixStepId = (idOrOptions: unknown, prefix: string): unknown => { if (typeof idOrOptions === "string") { - return `${prefix}${scopeSeparator}${idOrOptions}`; + return { + id: `${prefix}${scopeSeparator}${idOrOptions}`, + name: idOrOptions, + }; } if ( @@ -533,17 +542,57 @@ const prefixStepId = (idOrOptions: unknown, prefix: string): unknown => { return idOrOptions; }; +/** + * Run `fn` in a job's trace span. The span is top-level wherever the job is + * started from, so work on its machine that another job asks for later, like + * a snapshot, comes back to it. + */ +export const inJobSpan = ( + run: CiRunScope, + jobPath: string, + fn: () => R, +): R => { + return runWithAsyncCtx(run.asyncCtx, () => { + return inSpan( + { id: jobPath, name: traceName.job(run, jobPath), kind: "job" }, + fn, + ); + }); +}; + +/** + * Run `fn` outside every job's span, wherever it's called from. Jobs that + * start from one parent share its build, so the build's steps belong to none + * of them, and which one asks first must not decide where they sit in the + * trace. + */ +export const outsideJobs = (run: CiRunScope, fn: () => R): R => { + return runWithAsyncCtx(run.asyncCtx, fn); +}; + +/** + * Run `fn` in the run's one top-level GitHub span, wherever it's called from. + * Reporting isn't a job's own work, so a job's span ends with the job, and a + * failed job doesn't end on a green check update. + */ +export const inGitHubSpan = (run: CiRunScope, fn: () => R): R => { + return runWithAsyncCtx(run.asyncCtx, () => { + return inSpan(ciSpan("github", traceName.github), fn); + }); +}; + /** * Run a job handler inside both the CI job scope and a copy of the SDK's async - * context whose `step` prefixes IDs with the job path. + * context whose `step` prefixes IDs with the job path. It copies the context + * `inJobSpan` set, so the handler's steps are in the job's span. */ export const runJobBody = async ( scope: CiJobScope, fn: () => Promise, ): Promise => { - const { asyncCtx } = scope.run; + const asyncCtx = await getAsyncCtx(); - if (!asyncCtx.execution) { + if (!asyncCtx?.execution) { return runInScope({ run: scope.run, job: scope }, fn); } diff --git a/packages/ci/src/pipeline/spans.test.ts b/packages/ci/src/pipeline/spans.test.ts new file mode 100644 index 000000000..47fd17a5d --- /dev/null +++ b/packages/ci/src/pipeline/spans.test.ts @@ -0,0 +1,146 @@ +/** + * Tests of the span helpers, and of CI on an SDK without the span API: with + * none, functions still run, step IDs and names are exactly what they are + * with it, and the trace just has no spans or origins. The rest of the suite + * runs with the test stub in place. + * + * @module + */ + +import { group, step } from "inngest"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { consoleReporter } from "../github/auth.ts"; +import { $ } from "../machine/command.ts"; +import { createCiTestClient } from "../testing/client.ts"; +import { prEvent } from "../testing/events.ts"; +import { createFakeSandboxApi } from "../testing/fakeSandbox.ts"; +import { runFunction } from "../testing/runFunction.ts"; +import { installSpanStub, removeSpanStub } from "../testing/spanStub.ts"; +import { createCi } from "./createCi.ts"; +import { hasSpanApi, inSpan, originOption } from "./spans.ts"; + +/** Run a job with a command and a step of your own. */ +const run = () => { + const ci = createCi(createCiTestClient(createFakeSandboxApi()), { + github: consoleReporter(), + }); + + const job: () => Promise = ci.job("test", async () => { + await $`pnpm test`; + + await step.run("mine", () => { + return "done"; + }); + }); + + const pipeline = ci.pipeline( + { id: "pr", on: [{ event: "github/pull_request.opened" }] }, + job, + ); + + return runFunction(pipeline, { event: prEvent }); +}; + +describe("without the SDK's span API", () => { + beforeEach(() => { + removeSpanStub(); + }); + + afterEach(() => { + installSpanStub(); + }); + + test("is detected as absent", () => { + expect(hasSpanApi()).toBe(false); + + expect("~span" in group).toBe(false); + }); + + test("inSpan runs the function and returns what it returns", async () => { + const span = { id: "a", name: "A" }; + + expect(inSpan(span, () => 7)).toBe(7); + + await expect( + inSpan(span, () => { + return Promise.resolve("later"); + }), + ).resolves.toBe("later"); + + expect(() => { + return inSpan(span, () => { + throw new Error("from fn"); + }); + }).toThrow("from fn"); + }); + + test("steps keep their IDs and names, and the trace has no spans", async () => { + installSpanStub(); + + const withSpans = await run(); + + removeSpanStub(); + + const without = await run(); + + expect(without.type).toBe("function-resolved"); + + expect(without.stepIds).toEqual(withSpans.stepIds); + expect(without.names).toEqual(withSpans.names); + + expect(without.stepIds).toContain("test › pnpm test › start"); + expect(without.names["test › pnpm test › start"]).toBe("Start process"); + + expect(Object.keys(withSpans.spans).length).toBeGreaterThan(0); + expect(without.spans).toEqual({}); + + // Origins aren't asserted absent: an SDK with the span API stamps a + // step's `~origin` option itself, whether or not `group["~span"]` is there + // to open spans, and this suite runs against that SDK. + }); +}); + +describe("with the SDK's span API", () => { + test("is detected, and inSpan hands it the span and the function", () => { + expect(hasSpanApi()).toBe(true); + + removeSpanStub(); + + const calls: unknown[] = []; + + Object.defineProperty(group, "~span", { + value: (span: unknown, fn: () => unknown) => { + calls.push(span); + + return `wrapped ${String(fn())}`; + }, + configurable: true, + }); + + try { + const span = { id: "a", name: "A", kind: "job" }; + + expect( + inSpan(span, () => { + return "inner"; + }), + ).toBe("wrapped inner"); + + expect(calls).toEqual([span]); + } finally { + Reflect.deleteProperty(group, "~span"); + + installSpanStub(); + } + }); +}); + +describe("originOption", () => { + test("is an extra key on the step options, and nothing else", () => { + expect({ id: "a", name: "A", ...originOption("x") }).toEqual({ + id: "a", + name: "A", + "~origin": "x", + }); + }); +}); diff --git a/packages/ci/src/pipeline/spans.ts b/packages/ci/src/pipeline/spans.ts new file mode 100644 index 000000000..59872b69f --- /dev/null +++ b/packages/ci/src/pipeline/spans.ts @@ -0,0 +1,70 @@ +/** + * The one place CI touches the SDK's experimental trace-span API. The span + * API ships in newer SDKs than the oldest one CI supports, so it's + * feature-detected here: where the SDK has it, spans and origins mark up the + * trace, and where it doesn't, `inSpan` just runs the function and the origin + * step option is an unknown key the SDK ignores. Step IDs and names never + * depend on it. Everything else in CI goes through these helpers rather than + * reaching for the SDK's span API itself. + * + * @module + */ + +import { group } from "inngest"; + +/** A trace span, as the SDK's span API takes it. */ +export interface SpanInfo { + /** The span's stable ID, from which its rows are grouped. */ + id: string; + /** What the span is called in the trace. */ + name: string; + /** What kind of span it is, as in `job`. */ + kind?: string; + /** Who did the work in it, inherited by the steps and spans inside. */ + origin?: string; +} + +/** The part of the SDK's `group` this file uses, when the SDK has it. */ +interface SpanApi { + "~span"?: (span: SpanInfo, fn: () => R) => R; +} + +const spanApi = (): SpanApi => { + return group as unknown as SpanApi; +}; + +/** Whether the SDK in use has the span API. */ +export const hasSpanApi = (): boolean => { + return typeof spanApi()["~span"] === "function"; +}; + +/** + * Run `fn` in a trace span, or just run it when the SDK has no span API. + */ +export const inSpan = ( + /** The span to run `fn` in. */ + span: SpanInfo, + /** What to run. */ + fn: () => R, +): R => { + const api = spanApi(); + const open = api["~span"]; + + if (typeof open !== "function") { + return fn(); + } + + return open.call(api, span, fn) as R; +}; + +/** + * The step option that marks a step as work `origin` did. A step option is + * just a key on the options object, and an SDK without the span API ignores + * it, so it's safe to always pass. + */ +export const originOption = ( + /** Who did the work. */ + origin: string, +): { "~origin": string } => { + return { "~origin": origin }; +}; diff --git a/packages/ci/src/report.ts b/packages/ci/src/report.ts index 237a10650..c66c13230 100644 --- a/packages/ci/src/report.ts +++ b/packages/ci/src/report.ts @@ -4,6 +4,7 @@ * @module */ +import { traceName } from "./pipeline/names.ts"; import { countApi, getJobScope, @@ -43,7 +44,7 @@ export const report = { const id = nextStepId(run, job?.path, "report:summary"); - await run.step.run({ id, name: id }, () => { + await run.step.run({ id, name: traceName.addSummary }, () => { return { length: markdown.length, }; @@ -94,7 +95,7 @@ export const report = { const id = nextStepId(run, job?.path, "report:annotate"); - await run.step.run({ id, name: id }, () => { + await run.step.run({ id, name: traceName.addAnnotations }, () => { return { count: valid.length, }; diff --git a/packages/ci/src/testing/README.md b/packages/ci/src/testing/README.md index 688f4b139..7893f48ba 100644 --- a/packages/ci/src/testing/README.md +++ b/packages/ci/src/testing/README.md @@ -6,5 +6,6 @@ Test-only helpers. Not exported and not built. - `fakeGitHub.ts`: a fake GitHub HTTP layer. - `client.ts`: an Inngest client wired to the fake sandbox API, which also records the functions it creates. - `events.ts`: the pull request event and trigger tests share. +- `spanStub.ts`, `setup.ts`: a stand-in for the SDK's span API, installed before every test file when the SDK has none, so span and origin assertions hold on any SDK. - `runFunction.ts`: drives a function to completion like the executor, including `step.invoke` of the functions the test client created. Invokes run beside the function, and the function is called again as each one ends; runs of one `event.data` concurrency key take turns. - `schema.ts`: a fake Standard Schema that can also write itself as JSON Schema. diff --git a/packages/ci/src/testing/runFunction.ts b/packages/ci/src/testing/runFunction.ts index 805bee977..52b6f1085 100644 --- a/packages/ci/src/testing/runFunction.ts +++ b/packages/ci/src/testing/runFunction.ts @@ -15,6 +15,7 @@ import type { EventPayload, InngestFunction } from "inngest"; import { createdFunctions } from "./client.ts"; +import { SpanStubMiddleware, spanStubInstalled, stampOf } from "./spanStub.ts"; /** The opcodes the harness reads; their values are the wire format. */ const StepOpCode = { @@ -41,11 +42,19 @@ interface Step { data?: unknown; error?: unknown; metadata?: MetadataUpdate[]; - opts?: unknown; + opts?: { span?: StepSpanPath; origin?: string }; /** The step's ID before hashing. */ userland?: { id: string }; } +/** The spans a step is grouped under, outermost first. */ +type StepSpanPath = Array<{ + id: string; + name: string; + kind?: string; + origin?: string; +}>; + /** One execution request's outcome, loosely typed: the SDK doesn't export it. */ interface ExecutionResult { type: string; @@ -73,6 +82,10 @@ export interface RunResult { names: Record; /** Step data keyed by step ID. */ steps: Record; + /** The span path of each step in a span, keyed by step ID. */ + spans: Record; + /** The origin of each step that has one, keyed by step ID. */ + origins: Record; /** * What each `step.run` step was planned with, keyed by step ID: the options * the SDK sent the executor, such as a sandbox call's input. @@ -288,6 +301,8 @@ export const runFunction = async ( const batches: string[][] = []; const names: Record = {}; const steps: Record = {}; + const spans: RunResult["spans"] = {}; + const origins: RunResult["origins"] = {}; const inputs: Record = {}; const metadata: RunResult["metadata"] = []; @@ -324,6 +339,21 @@ export const runFunction = async ( names[id] = step.displayName ?? step.name ?? id; steps[id] = step.data; + + // Where the SDK has the span API it stamps these on the step. Where the + // test stub provides it, the stub recorded them instead. + const stamp = spanStubInstalled() ? stampOf(id) : undefined; + const span = + step.opts?.span ?? (stamp?.span.length ? stamp.span : undefined); + const origin = step.opts?.origin ?? stamp?.origin; + + if (span) { + spans[id] = span; + } + + if (origin) { + origins[id] = origin; + } }; // The executor retries a step that failed retriably, and only writes the @@ -382,6 +412,7 @@ export const runFunction = async ( ...(planned.displayName === undefined ? {} : { displayName: planned.displayName }), + opts: planned.opts, ...(outcome?.error === undefined ? { data: outcome?.data ?? null } : { error: outcome.error }), @@ -404,6 +435,8 @@ export const runFunction = async ( batches, names, steps, + spans, + origins, inputs, metadata, }; @@ -424,6 +457,8 @@ export const runFunction = async ( batches, names, steps, + spans, + origins, inputs, metadata, }; @@ -490,6 +525,7 @@ export const runFunction = async ( ? {} : { displayName: planned.displayName }), data: opts.resolveWait ? opts.resolveWait(planned) : null, + opts: planned.opts, }); continue; @@ -544,6 +580,11 @@ const runOnce = async ( return new Middleware({ client }); }); + if (spanStubInstalled()) { + // First, so the other middleware builds on step tools that record. + middlewareInstances.unshift(new SpanStubMiddleware({ client })); + } + const execution = internals["createExecution"]({ partialOptions: { client, diff --git a/packages/ci/src/testing/setup.ts b/packages/ci/src/testing/setup.ts new file mode 100644 index 000000000..f7532a0aa --- /dev/null +++ b/packages/ci/src/testing/setup.ts @@ -0,0 +1,11 @@ +/** + * Runs before every test file: gives the SDK the span API when it has none, + * so span and origin assertions hold against any SDK. A test of the no-span + * path takes it off again with `removeSpanStub`. + * + * @module + */ + +import { installSpanStub } from "./spanStub.ts"; + +installSpanStub(); diff --git a/packages/ci/src/testing/spanStub.ts b/packages/ci/src/testing/spanStub.ts new file mode 100644 index 000000000..481979fbe --- /dev/null +++ b/packages/ci/src/testing/spanStub.ts @@ -0,0 +1,185 @@ +/** + * A stand-in for the SDK's experimental trace-span API, for tests that run + * against an SDK that doesn't have it yet. It does what the real one does + * where tests can see it: a span opened with `inSpan` is kept in the async + * context, so a step called inside it is recorded under the span path, and a + * step's origin is its own `"~origin"` option or the innermost span's. Where + * the real SDK stamps a step's `opts.span` and `opts.origin` onto the step + * the executor reports, the stub records them by step ID for `runFunction` to + * read back. It installs nothing when the SDK has the span API itself. + * + * @module + */ + +import { group, Middleware } from "inngest"; +import type { AsyncContext } from "inngest/experimental"; +import { runWithAsyncCtx } from "inngest/experimental"; +import type { SpanInfo } from "../pipeline/spans.ts"; +import { hasSpanApi } from "../pipeline/spans.ts"; + +/** What the stub knows about a step: its spans, outermost first, and origin. */ +export interface StepStamp { + span: SpanInfo[]; + origin?: string; +} + +/** Where a span stack lives in an async context. Contexts copied by spread keep it. */ +const spanStackKey = Symbol("ci.spanStub.spans"); + +interface SpannedContext extends AsyncContext { + [spanStackKey]?: SpanInfo[]; +} + +const stamps = new Map(); + +let installed = false; + +/** + * The current async context, read synchronously like the real span API does. + * The SDK keeps its async storage on a global, and only offers an async getter, + * which would delay every span by a tick and reorder concurrent work. + */ +const currentContext = (): SpannedContext | undefined => { + const cache = ( + globalThis as Record< + symbol, + { resolved?: { getStore: () => AsyncContext | undefined } } | undefined + > + )[Symbol.for("inngest:als")]; + + return cache?.resolved?.getStore(); +}; + +const spanOf = (span: SpanInfo, fn: () => R): R => { + const ctx = currentContext(); + + if (!ctx) { + return fn(); + } + + const spans = [...(ctx[spanStackKey] ?? []), span]; + + const spanned: SpannedContext = { ...ctx, [spanStackKey]: spans }; + + return runWithAsyncCtx(spanned, fn); +}; + +/** The SDK's own `"~span"`, while `removeSpanStub` has it taken off. */ +let hidden: PropertyDescriptor | undefined; + +/** Give the SDK's `group` a `"~span"`, unless it already has one. */ +export const installSpanStub = (): void => { + if (hidden) { + Object.defineProperty(group, "~span", hidden); + + hidden = undefined; + + return; + } + + if (hasSpanApi()) { + return; + } + + Object.defineProperty(group, "~span", { + value: spanOf, + configurable: true, + writable: true, + }); + + installed = true; +}; + +/** + * Take the stub off again, so the SDK looks as it does without the span API. + * An SDK that has the span API itself has it taken off too, until + * `installSpanStub` puts it back. + */ +export const removeSpanStub = (): void => { + if (installed) { + Reflect.deleteProperty(group, "~span"); + + installed = false; + + return; + } + + hidden ??= Object.getOwnPropertyDescriptor(group, "~span"); + + Reflect.deleteProperty(group, "~span"); +}; + +/** Whether the stub is what's providing the span API. */ +export const spanStubInstalled = (): boolean => { + return installed; +}; + +/** What the stub recorded for a step, by the ID CI gave it. */ +export const stampOf = (stepId: string): StepStamp | undefined => { + return stamps.get(stepId); +}; + +/** Record the step `args` is about to run, from the span stack in effect now. */ +const record = (args: unknown[]): void => { + const first = args[0]; + + const options = + typeof first === "string" + ? { id: first } + : (first as { id?: unknown; "~origin"?: unknown } | undefined); + + if (typeof options?.id !== "string") { + return; + } + + const span = currentContext()?.[spanStackKey] ?? []; + + const inherited = [...span].reverse().find((entry) => { + return entry.origin; + })?.origin; + + const own = (options as { "~origin"?: unknown })["~origin"]; + + stamps.set(options.id, { + span, + origin: typeof own === "string" ? own : inherited, + }); +}; + +/** + * Wrap step tools so every call is recorded. The SDK's middleware hooks don't + * hand a step's unknown options to middleware, so the calls are watched where + * they're made. + */ +const recording = (tools: T): T => { + return new Proxy(tools, { + get(target, prop, receiver) { + const value = Reflect.get(target, prop, receiver) as unknown; + + if (typeof value === "function") { + return (...args: unknown[]) => { + record(args); + + return (value as (...a: unknown[]) => unknown).apply(target, args); + }; + } + + if (value && typeof value === "object") { + return recording(value); + } + + return value; + }, + }); +}; + +/** Records each step's span path and origin as it's called. */ +export class SpanStubMiddleware extends Middleware.BaseMiddleware { + readonly id = "ci-span-stub"; + + override transformFunctionInput( + arg: Middleware.TransformFunctionInputArgs, + ): Middleware.TransformFunctionInputArgs { + return { ...arg, ctx: { ...arg.ctx, step: recording(arg.ctx.step) } }; + } +} diff --git a/packages/ci/src/types.ts b/packages/ci/src/types.ts index 07ce386f5..a09e84dd0 100644 --- a/packages/ci/src/types.ts +++ b/packages/ci/src/types.ts @@ -202,7 +202,7 @@ export interface CacheConfig { * ``` */ export interface JobConfig { - /** Unique within the CI client. It's the job's path in the trace and its check name. */ + /** Unique within the CI client. It's the job's check name, and its step IDs start with it. */ id: string; /** * The job to start from: this job runs on a copy of that job's machine, so @@ -232,6 +232,8 @@ export interface JobConfig { * choose between them in the pipeline. */ from?: From; + /** What the trace calls the job. Defaults to `id`. */ + name?: string; /** Machine settings for this job, overriding the pipeline's and the client's. */ machine?: MachineConfig; /** diff --git a/packages/ci/src/units.test.ts b/packages/ci/src/units.test.ts index 23dcfa85c..c15e89fe0 100644 --- a/packages/ci/src/units.test.ts +++ b/packages/ci/src/units.test.ts @@ -710,6 +710,7 @@ describe("looking a cached snapshot up by name", () => { get: () => Promise = async () => { return null; }, + exclude?: string, ) => { const scope = { path: "setup", @@ -728,6 +729,7 @@ describe("looking a cached snapshot up by name", () => { scope, { key: "v1", scope: "global" }, { ownKey: "k1", name }, + exclude, ); }; @@ -793,6 +795,12 @@ describe("looking a cached snapshot up by name", () => { createdAt: snapshot({}).createdAt, }); }); + + test("a snapshot found to be bad is never found again", async () => { + expect(await lookup(listing(snapshot({})), undefined, "s1")).toBe( + undefined, + ); + }); }); describe("snapshot names", () => { diff --git a/packages/ci/vitest.config.ts b/packages/ci/vitest.config.ts index 2a6163f50..d3b49e53c 100644 --- a/packages/ci/vitest.config.ts +++ b/packages/ci/vitest.config.ts @@ -4,6 +4,7 @@ export default defineConfig({ test: { environment: "node", silent: "passed-only", + setupFiles: ["src/testing/setup.ts"], typecheck: { enabled: true, include: ["src/**/*.test.ts"], From ea84b11c3d1d540aa58991f8d6a209b5edc840be Mon Sep 17 00:00:00 2001 From: x Date: Sat, 10 Oct 2026 00:41:01 +0000 Subject: [PATCH 02/11] refactor(ci): span catalog, staged machine start, span kinds folded in Every span is an entry in the `spans` catalog with a required kind, step IDs are built by `joinId`, and starting a machine is a flat staged function in `machine/start.ts`. Folds the CI half of ci2-span-kinds: commands, sandboxes, saves and attempts carry kinds, each statement on a background process is a row of its own, and a snapshot's span is passed to an SDK that takes it (feature-detected by version) and opened by CI otherwise. Names the checkout steps. Adds a `ciTest` harness. Co-Authored-By: Claude Opus 5.5 --- .changeset/ci-trace-spans.md | 2 +- packages/ci/src/cache/cache.ts | 8 +- packages/ci/src/checkout/checkout.ts | 27 +- packages/ci/src/machine/command.ts | 108 +++-- packages/ci/src/machine/from.ts | 4 +- packages/ci/src/machine/machine.ts | 459 ++---------------- packages/ci/src/machine/sandbox.ts | 8 +- packages/ci/src/machine/sandboxErrors.ts | 67 +++ packages/ci/src/machine/spans.test.ts | 379 ++++++--------- packages/ci/src/machine/start.ts | 335 +++++++++++++ .../ci/src/pipeline/cacheGuarantees.test.ts | 2 +- packages/ci/src/pipeline/job.ts | 7 +- packages/ci/src/pipeline/names.ts | 72 ++- packages/ci/src/pipeline/pipeline.test.ts | 7 +- packages/ci/src/pipeline/scope.ts | 20 +- packages/ci/src/pipeline/spans.test.ts | 169 +++++-- packages/ci/src/pipeline/spans.ts | 61 ++- packages/ci/src/testing/README.md | 1 + packages/ci/src/testing/harness.ts | 115 +++++ 19 files changed, 1058 insertions(+), 793 deletions(-) create mode 100644 packages/ci/src/machine/sandboxErrors.ts create mode 100644 packages/ci/src/machine/start.ts create mode 100644 packages/ci/src/testing/harness.ts diff --git a/.changeset/ci-trace-spans.md b/.changeset/ci-trace-spans.md index d3ef514f5..abe6dd96e 100644 --- a/.changeset/ci-trace-spans.md +++ b/.changeset/ci-trace-spans.md @@ -2,4 +2,4 @@ "@inngest/ci": minor --- -Group a run's steps into trace spans (one per job, one per command, one per extra machine), name CI's own steps for what they do, and mark them as CI's work so a trace can tell them from yours. Give a job a `name` to change what its span is called. +Group a run's steps into trace spans (one per job, one per command, one per extra machine), name CI's own steps for what they do, and mark them as CI's work so a trace can tell them from yours. Give a job a `name` to change what its span is called. Every span has a kind (job, sandbox, command, snapshot, github or attempt), and each statement on a background process (`output`, `exited`, `kill`) is a row of its own. Saving a sandbox is one `snapshot` row on an `inngest` that takes the span, and the span is opened by CI on older ones. diff --git a/packages/ci/src/cache/cache.ts b/packages/ci/src/cache/cache.ts index d53d856b0..173b327c5 100644 --- a/packages/ci/src/cache/cache.ts +++ b/packages/ci/src/cache/cache.ts @@ -13,7 +13,7 @@ import { tagStep } from "../pipeline/metadata.ts"; import { ciStep, traceName } from "../pipeline/names.ts"; import type { CiJobScope, CiRunScope } from "../pipeline/scope.ts"; -import { countApi, rootRunIdOf, scopeSeparator } from "../pipeline/scope.ts"; +import { countApi, joinId, rootRunIdOf } from "../pipeline/scope.ts"; import type { CacheConfig, CacheKey, @@ -283,7 +283,7 @@ export const cacheTarget = async ( countApi("cache"); const ownKey = (await run.step.run( - ciStep(`${job.path}${scopeSeparator}cache:key`, traceName.checkCache), + ciStep(joinId(job.path, "cache:key"), traceName.checkCache), async () => { await tagStep(run, { kind: "cache", job: job.path }); @@ -462,7 +462,7 @@ export const lookupCache = async ( const { run } = scope; const found = (await run.step.run( - ciStep(`${scope.path}${scopeSeparator}cache:lookup`, traceName.lookUpCache), + ciStep(joinId(scope.path, "cache:lookup"), traceName.lookUpCache), async () => { await tagStep(run, { kind: "cache", job: scope.path }); @@ -516,7 +516,7 @@ export const lookupBeforeBuild = async ( exclude?: string, ): Promise => { const found = (await run.step.run( - ciStep(`${job.stepPath}${scopeSeparator}lookup`, traceName.lookUpCache), + ciStep(joinId(job.stepPath, "lookup"), traceName.lookUpCache), async () => { await tagStep(run, { kind: "cache", job: job.path }); diff --git a/packages/ci/src/checkout/checkout.ts b/packages/ci/src/checkout/checkout.ts index 1914f6a0b..cb8256309 100644 --- a/packages/ci/src/checkout/checkout.ts +++ b/packages/ci/src/checkout/checkout.ts @@ -7,12 +7,13 @@ import { CiUsageError } from "../errors.ts"; import { ensureMachine } from "../machine/machine.ts"; +import { traceName } from "../pipeline/names.ts"; import type { CiRunScope, MachineHandle } from "../pipeline/scope.ts"; import { countApi, defaultCwd, + joinId, requireJobScope, - scopeSeparator, } from "../pipeline/scope.ts"; import type { RepoContext } from "../types.ts"; import { shellEscape } from "../util.ts"; @@ -88,15 +89,21 @@ export const checkout = async (opts: CheckoutOptions = {}): Promise => { } const machine = await ensureMachine(scope); - const stepId = `${scope.path}${scopeSeparator}checkout`; - - await run.step.run({ id: stepId, name: stepId }, async () => { - if (local) { - return uploadWorkingTree(run, machine, local.path, target); - } - - return cloneFromGithub(run, machine, repo as RepoContext, opts, target); - }); + const stepId = joinId(scope.path, "checkout"); + + await run.step.run( + { + id: stepId, + name: local ? traceName.uploadWorkingTree : traceName.cloneRepository, + }, + async () => { + if (local) { + return uploadWorkingTree(run, machine, local.path, target); + } + + return cloneFromGithub(run, machine, repo as RepoContext, opts, target); + }, + ); scope.cwd ??= target; }; diff --git a/packages/ci/src/machine/command.ts b/packages/ci/src/machine/command.ts index 6e0a0f732..7cf420bee 100644 --- a/packages/ci/src/machine/command.ts +++ b/packages/ci/src/machine/command.ts @@ -13,14 +13,14 @@ import { CommandFailedError, CommandTimeoutError, } from "../errors.ts"; -import { ciSpan, ciStep, traceName } from "../pipeline/names.ts"; +import { ciStep, spans, traceName } from "../pipeline/names.ts"; import type { CiJobScope, MachineHandle } from "../pipeline/scope.ts"; import { countApi, defaultCwd, + joinId, nextStepId, requireJobScope, - scopeSeparator, } from "../pipeline/scope.ts"; import { inSpan } from "../pipeline/spans.ts"; import type { @@ -176,7 +176,7 @@ export const buildShellString = ( * command's span is yours, but how CI runs it isn't. */ const subStep = (stepId: string, suffix: string, name: string) => { - return ciStep(`${stepId}${scopeSeparator}${suffix}`, name); + return ciStep(joinId(stepId, suffix), name); }; class CommandBuilder implements Command { @@ -268,7 +268,8 @@ class CommandBuilder implements Command { const stepId = this.stepId(scope); const machine = await ensureMachine(scope); - // Each call back into the process re-enters the command's span. + // Starting, awaiting, reading and killing the process are separate + // statements, so each is a row of its own, never one span re-entered. const process = await this.inSpan(scope, stepId, () => { return startProcess(machine, stepId, this.spawnOptions(scope)); }); @@ -278,39 +279,60 @@ class CommandBuilder implements Command { // Numbers the follow-up steps, so calling `exited()` or `output()` more // than once never reuses a step ID. const nextWait = counter(); + const nextExited = counter(); return { id: process.id, exited: () => { - return this.inSpan(scope, stepId, async () => { - const polled = await pollUntilTerminal({ - scope, - machine, - process, - stepId, - nextWait, - }); - - return readResult({ process: polled.process, stepId, argv, secrets }); - }); + return this.inSpan( + scope, + stepId, + async () => { + const polled = await pollUntilTerminal({ + scope, + machine, + process, + stepId, + nextWait, + }); + + return readResult({ + process: polled.process, + stepId, + argv, + secrets, + }); + }, + { suffix: `exited #${nextExited()}`, label: "exited" }, + ); }, kill: async (signal = 15) => { - await this.inSpan(scope, stepId, () => { - return process.signal( - subStep(stepId, "kill", traceName.stopProcess), - { signal }, - ); - }); + await this.inSpan( + scope, + stepId, + () => { + return process.signal( + subStep(stepId, "kill", traceName.stopProcess), + { signal }, + ); + }, + { suffix: "kill", label: "kill" }, + ); }, output: async (opts) => { - const output = await this.inSpan(scope, stepId, () => { - return process.getOutput( - subStep(stepId, `output #${nextWait()}`, traceName.readOutput), - { - tailBytes: opts?.tailBytes ?? outputTailBytes, - }, - ); - }); + const suffix = `output #${nextWait()}`; + + const output = await this.inSpan( + scope, + stepId, + () => { + return process.getOutput( + subStep(stepId, suffix, traceName.readOutput), + { tailBytes: opts?.tailBytes ?? outputTailBytes }, + ); + }, + { suffix, label: "output" }, + ); const decoded = decodeChunks(output); @@ -371,12 +393,23 @@ class CommandBuilder implements Command { }); } - /** Run `fn` in the command's span, in its machine's span if it's an extra. */ - private inSpan(scope: CiJobScope, stepId: string, fn: () => T): T { - const span = { - id: stepId, - name: traceName.command(this.commandText(), this.state.label), - }; + /** + * Run `fn` in the command's span, in its machine's span if it's an extra. + * A statement on a background process, such as its `kill`, is a span of its + * own beside the command's. + */ + private inSpan( + scope: CiJobScope, + stepId: string, + fn: () => T, + statement?: { suffix: string; label: string }, + ): T { + const span = spans.command( + stepId, + this.commandText(), + this.state.label, + statement, + ); return inMachineSpan(scope, () => { return inSpan(span, fn); @@ -413,10 +446,7 @@ class CommandBuilder implements Command { const result = await this.inSpan(scope, stepId, () => { return attempts === 1 ? runAttempt() - : inSpan( - ciSpan(`attempt-${attempt}`, traceName.attempt(attempt)), - runAttempt, - ); + : inSpan(spans.attempt(attempt), runAttempt); }); scope.run.ci.hooks.commandFinished(scope, attemptInfo, result); diff --git a/packages/ci/src/machine/from.ts b/packages/ci/src/machine/from.ts index 7c1a7ed03..5a1bd1066 100644 --- a/packages/ci/src/machine/from.ts +++ b/packages/ci/src/machine/from.ts @@ -25,9 +25,9 @@ import { adoptBuilt, invokeBuild, validateInput } from "../pipeline/job.ts"; import type { CiJobScope, CiRunScope } from "../pipeline/scope.ts"; import { countApi, + joinId, outsideJobs, rebuildSuffix, - scopeSeparator, } from "../pipeline/scope.ts"; import type { AnyJob, JobConfig, JobRef } from "../types.ts"; import { errorMessage, hash, stableStringify } from "../util.ts"; @@ -486,7 +486,7 @@ const requestRebuild = ({ if (replacing.broken) { const gone = await deleteSnapshot( run, - `${path}${scopeSeparator}cache:delete`, + joinId(path, "cache:delete"), replacing.snapshotId, ); diff --git a/packages/ci/src/machine/machine.ts b/packages/ci/src/machine/machine.ts index a0c4f6779..2a38b7a1e 100644 --- a/packages/ci/src/machine/machine.ts +++ b/packages/ci/src/machine/machine.ts @@ -7,51 +7,32 @@ import type { Inngest } from "inngest"; import type { CachedSnapshot, CacheTarget } from "../cache/cache.ts"; -import { resolveTakenName, snapshotState } from "../cache/cache.ts"; -import { CiUsageError } from "../errors.ts"; -import { ciSpan, ciStep, traceName } from "../pipeline/names.ts"; +import { resolveTakenName } from "../cache/cache.ts"; +import { ciStep, spans, traceName } from "../pipeline/names.ts"; import type { CiJobScope, CiRunScope, MachineHandle, } from "../pipeline/scope.ts"; import { - defaultCwd, inJobSpan, isRestoring, + joinId, runInScope, scopeSeparator, } from "../pipeline/scope.ts"; -import { inSpan } from "../pipeline/spans.ts"; -import type { MachineConfig } from "../types.ts"; import { - boundedName, + inSnapshotSpan, + inSpan, + snapshotSpanOption, +} from "../pipeline/spans.ts"; +import { errorMessage, isSandboxNotFound, isSnapshotNotFound, - slug, } from "../util.ts"; - -/** - * Memory is paired with vCPU count, so a job only picks one number. - */ -const memoryForVcpu = { 1: 1024, 2: 2048, 4: 4096 } as const; - -export const resolveMachineConfig = ( - config: MachineConfig | undefined, -): { vcpu: 1 | 2 | 4; memoryMb: number } => { - const vcpu = config?.vcpu ?? 2; - - return { vcpu, memoryMb: memoryForVcpu[vcpu] }; -}; - -/** - * The name a machine is created with. It carries the run ID so orphaned - * machines can be found and destroyed by the cleanup function. - */ -export const machineName = (runId: string, path: string): string => { - return boundedName(`ci-${runId}-${slug(path)}`); -}; +import { errorStatus, hasCode } from "./sandboxErrors.ts"; +import { startMachine } from "./start.ts"; /** * Create this scope's machine if it doesn't have one yet. @@ -65,14 +46,14 @@ export const ensureMachine = async ( ): Promise => { scope.machine ??= inJobSpan(scope.run, scope.jobPath, () => { return inMachineSpan(scope, () => { - const stepId = `${scope.path}${scopeSeparator}machine`; + const stepId = joinId(scope.path, "machine"); // Everything in it is CI's work, including falling back from a // snapshot that won't start. - const span = ciSpan(stepId, traceName.startMachine(scope.fromJobIds[0])); + const span = spans.startMachine(stepId, scope.fromJobIds[0]); return inSpan(span, () => { - return createMachine(scope, stepId); + return startMachine(scope, stepId); }); }); }); @@ -109,391 +90,8 @@ export const inMachineSpan = (scope: CiJobScope, fn: () => R): R => { return fn(); } - const span = { - id: scope.path, - name: traceName.extraMachine(scope.path, scope.jobPath), - }; - - return inSpan(span, fn); -}; - -/** - * Run once on every machine before its first command. - * - * WORKAROUNDS (Sandboxes API), delete each part once the platform covers it: - * - Commands run in `/work` by default, and a sandbox won't start a process in - * a directory that doesn't exist, so it's created here. `checkout()` also - * makes it, but a job doesn't have to check out. - * - Sandboxes boot with the loopback interface down, so nothing can listen on - * or reach `127.0.0.1`, which breaks services started with `.background()` - * and `waitForHttp`/`waitForPort`. Bringing it up is a no-op once the - * platform does it itself. - */ -export const machineSetupScript = `mkdir -p ${defaultCwd} && (ip link set lo up 2>/dev/null || true)`; - -/** A machine that has started and been set up. */ -interface Started { - // biome-ignore lint/suspicious/noExplicitAny: DurableSandbox - sandbox: any; -} - -const createMachine = async ( - scope: CiJobScope, - /** The create step's ID, which other steps for the machine build on. */ - stepId: string, -): Promise => { - const { run } = scope; - const tools = run.sandboxTools; - - if (!tools) { - throw new CiUsageError( - "This pipeline has no `step.sandbox` tools. `createCi` adds `sandboxMiddleware()` to the functions it creates, so this usually means the function was created another way.", - ); - } - - const name = machineName(run.runId, scope.path); - const create = { id: stepId, name: traceName.createMachine }; - - const machineConfig = resolveMachineConfig( - scope.config.machine ?? run.machine ?? run.ci.defaultMachine, - ); - - /** Create a machine, from a snapshot if one is given, and set it up. */ - const start = async ( - createStep: { id: string; name: string }, - options: { - name: string; - snapshotId?: string; - }, - ): Promise => { - const sandbox = options.snapshotId - ? await tools.create(createStep, { - name: options.name, - snapshotId: options.snapshotId, - }) - : await tools.create(createStep, { - name: options.name, - ...machineConfig, - }); - - await sandbox.commands.run( - { - id: `${createStep.id}${scopeSeparator}setup`, - name: traceName.prepareWorkspace, - }, - ["/bin/sh", "-c", machineSetupScript], - ); - - return { sandbox }; - }; - - const startFresh = (note: string) => { - scope.fromSnapshotId = undefined; - scope.startNote = note; - scope.restoreFallback = scope.rebuildParent; - - run.ci.hooks.activity(run, scope.jobPath, note); - - return start( - { - id: `${stepId}${scopeSeparator}fresh`, - name: traceName.createFreshMachine, - }, - { name: machineName(run.runId, `${scope.path} fresh`) }, - ); - }; - - const parentId = scope.fromJobIds[0] ?? "the parent"; - const snapshotId = scope.fromSnapshotId; - - let started: Started; - - if (snapshotId) { - // Every job starting from a snapshot checks it for itself, so what a job - // does never depends on whether a sibling got there first. Only the - // rebuild of the parent, which each of them awaits, is shared. - run.ci.hooks.activity( - run, - scope.jobPath, - scope.startNote ?? "creating machine…", - ); - - let probed: Started | undefined; - - /** Why the snapshot can't be used, if it can't. */ - let bad: string | undefined; - - /** How to replace it, when it can't be used. */ - let rebuild = { broken: false, unnamed: false }; - - try { - probed = await start(create, { name, snapshotId }); - } catch (error) { - if (!isStartFailure(error)) { - throw error; - } - - // The failed start still holds `name`, so what replaces it gets - // another, and the stuck machine is cleared away meanwhile. - await discardFailedStart(run, stepId, name, error); - - bad = `wouldn't start (${errorMessage(error)})`; - - // WORKAROUND (Sandboxes API): a snapshot can't be told to be broken, so - // it is retried before it is deleted. Delete this once snapshots or - // images report their own health. Today a snapshot has only the - // statuses `CREATING`, `READY`, `DELETING` and `DELETED`, and a broken - // one still says `READY`. A start that times out says no more: the - // machine may have been stuck, or the node busy, as easily as the - // snapshot bad. Deleting on the first one would let a single blip - // destroy a cache that is shared, since a pull request restores its - // base branch's snapshots, and every run after it would build again. So - // only a snapshot that fails to start a second time is broken enough to - // delete. Capacity errors (`compute_unavailable`, 429 and 503) are not - // start failures at all, and fail the job as any other would. - const state = await snapshotState( - run, - `${stepId}${scopeSeparator}snapshot-state`, - snapshotId, - ); - - if (state === "creating") { - // Another build is still taking it and may yet finish, so this - // run rebuilds without contending for its name. - rebuild = { broken: false, unnamed: true }; - } else if (state === "ready") { - const restartStepId = `${stepId}${scopeSeparator}restart`; - const restartName = machineName(run.runId, `${scope.path} restart`); - - try { - // With the default wait, the same as the first try's: a shorter one - // would make a slow node more likely to fail the retry, and a second - // failure deletes the snapshot. - probed = await start( - { id: restartStepId, name: traceName.restartMachine }, - { - name: restartName, - snapshotId, - }, - ); - - bad = undefined; - - run.warnings.push( - `retried: snapshot of \`${parentId}\` needed a retry to start (${errorMessage(error)}), so \`${scope.path}\` started it on the second try`, - ); - } catch (second) { - if (!isStartFailure(second)) { - throw second; - } - - await discardFailedStart(run, restartStepId, restartName, second); - - bad = `wouldn't start twice (${errorMessage(second)})`; - - rebuild = { broken: true, unnamed: false }; - } - } - } - - if (probed && !bad) { - started = probed; - } else { - run.warnings.push( - `fell back: snapshot of \`${parentId}\` ${bad}, so \`${scope.path}\` rebuilt it`, - ); - - const note = `rebuilding ${parentId} · bad snapshot`; - - scope.startNote = note; - - run.ci.hooks.activity(run, scope.jobPath, note); - - // Shared: jobs that had the same trouble wait for one rebuild, which - // also deletes a broken snapshot once. - const replacement = await scope.rebuildSnapshot?.(rebuild); - - if (replacement) { - scope.startNote = `starting ${parentId}`; - - run.ci.hooks.activity(run, scope.jobPath, scope.startNote); - - const retryStepId = `${stepId}${scopeSeparator}retry`; - const retryName = machineName(run.runId, `${scope.path} retry`); - - try { - started = await start( - { id: retryStepId, name: traceName.retryCreate }, - { - name: retryName, - snapshotId: replacement, - }, - ); - } catch (error) { - if (!isStartFailure(error)) { - throw error; - } - - // Restores may still be broken while fresh machines work, which is - // where this job went before there was a replacement to try. - await discardFailedStart(run, retryStepId, retryName, error); - - started = await startFresh(note); - } - } else { - started = await startFresh(note); - } - } - } else { - run.ci.hooks.activity( - run, - scope.jobPath, - scope.startNote ?? "creating machine…", - ); - - started = await start(create, { name }); - } - - const handle: MachineHandle = { - sandbox: started.sandbox, - name, - id: started.sandbox.id, - }; - - return handle; + return inSpan(spans.extraMachine(scope.path, scope.jobPath), fn); }; - -/** - * Best-effort cleanup of a machine that failed to start. The platform keeps - * such a machine (stuck in STARTING) and its name, so it is destroyed here, and - * the end of the run finds it by its name if this couldn't. - * - * Never throws: a machine that can't be cleaned up now must not fail the job. - */ -const discardFailedStart = async ( - run: CiRunScope, - stepId: string, - name: string, - error: unknown, -): Promise => { - const knownId = (error as { cause?: { sandboxId?: string } })?.cause - ?.sandboxId; - - try { - await run.step.run( - { - id: `${stepId}${scopeSeparator}discard`, - name: traceName.discardMachine, - }, - async (): Promise<{ id?: string }> => { - // Errors are swallowed inside the step so it never retries. - let id = knownId; - - try { - let cursor: string | undefined; - - while (!id) { - const page = await run.ci.client.sandboxes.list({ - ...(cursor ? { cursor } : {}), - limit: 100, - }); - - id = page.items.find( - (sandbox: { name: string; status: string }) => { - return sandbox.name === name && sandbox.status !== "TERMINATED"; - }, - )?.id; - - if (id || !page.page.hasMore) { - break; - } - - cursor = page.page.cursor; - } - - if (id) { - const sandbox = await run.ci.client.sandboxes.get(id); - - await sandbox?.destroy(); - } - } catch { - // The run's cleanup finds it again by its name. - } - - return id ? { id } : {}; - }, - ); - } catch { - // Best effort only. - } -}; - -/** - * Whether a machine failed to start, as opposed to a request that was refused. - * Only a snapshot restore falls back on it: a plain machine that won't start - * fails the job with its reason. - */ -const isStartFailure = (error: unknown): boolean => { - const codes = ["sandbox_start_timed_out", "sandbox_start_failed"]; - - // No capacity, or too many requests, says nothing about the snapshot. - if (isRetryable(error)) { - return false; - } - - return ( - codes.some((code) => { - return hasCode(error, code); - }) || /did not reach RUNNING/i.test(errorMessage(error)) - ); -}; - -/** - * Whether the platform says to try again, as it does for no capacity - * (`compute_unavailable`) and rate limits (429 and 503). - */ -const isRetryable = (error: unknown): boolean => { - const seen = error as - | { retryable?: boolean; cause?: { retryable?: boolean } } - | undefined; - const status = errorStatus(error); - - return ( - seen?.retryable === true || - seen?.cause?.retryable === true || - hasCode(error, "compute_unavailable") || - hasCode(error, "rate_limited") || - status === 429 || - status === 503 - ); -}; - -/** Whether a Sandboxes error has a code, on the error or the step error's cause. */ -const hasCode = (error: unknown, code: string): boolean => { - const seen = error as - | { code?: string; cause?: { code?: string } } - | undefined; - - return seen?.code === code || seen?.cause?.code === code; -}; - -/** A Sandboxes error's HTTP status, whether on the error or its cause. */ -const errorStatus = (error: unknown): number | undefined => { - const seen = error as - | { status?: number; cause?: { status?: number } } - | undefined; - - return seen?.status ?? seen?.cause?.status; -}; - -/** Run `fn` in a job's "Save sandbox" span, which holds snapshotting it. */ -const inSaveSpan = (jobPath: string, fn: () => R): R => { - return inSpan( - ciSpan(`${jobPath}${scopeSeparator}save`, traceName.saveMachine), - fn, - ); -}; - /** How a job's snapshot is named. */ export interface SnapshotCache { target: CacheTarget; @@ -523,9 +121,17 @@ export interface TakenSnapshot { reused: boolean; } -/** A snapshot step: its ID is the step's, its name reads as an action. */ -const snapshotStep = (id: string) => { - return { id, name: traceName.snapshotMachine }; +/** + * A snapshot step: its ID is the step's, its name reads as an action. Saving + * a job's sandbox is one statement, so its row is the job's "Save sandbox" + * span, which the SDK is told of where it can take it. + */ +const snapshotStep = (jobPath: string, id: string) => { + return { + id, + name: traceName.snapshotMachine, + ...snapshotSpanOption(spans.save(jobPath)), + }; }; /** @@ -549,7 +155,7 @@ export const snapshotMachine = async ( const handle = await scope.machine; - return inSaveSpan(scope.path, () => { + return inSnapshotSpan(spans.save(scope.path), () => { return takeSnapshot(run, scope.path, handle, cache); }); }; @@ -562,12 +168,12 @@ const takeSnapshot = async ( ): Promise => { run.ci.hooks.activity(run, jobPath, "snapshotting machine…"); - const stepId = `${jobPath}${scopeSeparator}snapshot`; + const stepId = joinId(jobPath, "snapshot"); try { return cache && !cache.unnamed ? await createNamedSnapshot(run, handle, jobPath, stepId, cache) - : await createRunSnapshot(handle, stepId); + : await createRunSnapshot(handle, jobPath, stepId); } catch (error) { if (!isSnapshotUnavailable(error)) { throw error; @@ -601,7 +207,9 @@ const createNamedSnapshot = async ( const name = target.name; const attempt = async (id: string): Promise => { - const snapshot = await handle.sandbox.snapshot(snapshotStep(id), { name }); + const snapshot = await handle.sandbox.snapshot(snapshotStep(jobPath, id), { + name, + }); return { snapshotId: snapshot.id, @@ -659,7 +267,7 @@ const createNamedSnapshot = async ( } const unnamed = await handle.sandbox.snapshot( - snapshotStep(`${stepId} (unnamed)`), + snapshotStep(jobPath, `${stepId} (unnamed)`), ); // The name was refused or couldn't be freed, so the job's snapshot falls @@ -676,9 +284,10 @@ const createNamedSnapshot = async ( */ const createRunSnapshot = async ( handle: MachineHandle, + jobPath: string, stepId: string, ): Promise => { - const snapshot = await handle.sandbox.snapshot(snapshotStep(stepId)); + const snapshot = await handle.sandbox.snapshot(snapshotStep(jobPath, stepId)); return { snapshotId: snapshot.id, reused: false }; }; diff --git a/packages/ci/src/machine/sandbox.ts b/packages/ci/src/machine/sandbox.ts index 616475ebd..f0caad9d5 100644 --- a/packages/ci/src/machine/sandbox.ts +++ b/packages/ci/src/machine/sandbox.ts @@ -6,11 +6,7 @@ import { waitForHttp, waitForPort } from "../checkout/wait.ts"; import type { CiJobScope } from "../pipeline/scope.ts"; -import { - countApi, - requireJobScope, - scopeSeparator, -} from "../pipeline/scope.ts"; +import { countApi, joinId, requireJobScope } from "../pipeline/scope.ts"; import type { Duration, ExtraMachine, MachineConfig } from "../types.ts"; import { createCommandTag } from "./command.ts"; import { ensureMachine } from "./machine.ts"; @@ -53,7 +49,7 @@ export const sandbox = async ( const job = requireJobScope("sandbox"); countApi("sandbox"); - const path = `${job.jobPath}${scopeSeparator}${name}`; + const path = joinId(job.jobPath, name); const existing = job.extras?.get(name); diff --git a/packages/ci/src/machine/sandboxErrors.ts b/packages/ci/src/machine/sandboxErrors.ts new file mode 100644 index 000000000..7b10821f8 --- /dev/null +++ b/packages/ci/src/machine/sandboxErrors.ts @@ -0,0 +1,67 @@ +/** + * What a Sandboxes error says about itself: whether a machine failed to + * start, whether the platform says to try again, and the code and status it + * carries, on the error or on the step error's cause. + * + * @module + */ + +import { errorMessage } from "../util.ts"; + +/** + * Whether a machine failed to start, as opposed to a request that was refused. + * Only a snapshot restore falls back on it: a plain machine that won't start + * fails the job with its reason. + */ +export const isStartFailure = (error: unknown): boolean => { + const codes = ["sandbox_start_timed_out", "sandbox_start_failed"]; + + // No capacity, or too many requests, says nothing about the snapshot. + if (isRetryable(error)) { + return false; + } + + return ( + codes.some((code) => { + return hasCode(error, code); + }) || /did not reach RUNNING/i.test(errorMessage(error)) + ); +}; + +/** + * Whether the platform says to try again, as it does for no capacity + * (`compute_unavailable`) and rate limits (429 and 503). + */ +const isRetryable = (error: unknown): boolean => { + const seen = error as + | { retryable?: boolean; cause?: { retryable?: boolean } } + | undefined; + const status = errorStatus(error); + + return ( + seen?.retryable === true || + seen?.cause?.retryable === true || + hasCode(error, "compute_unavailable") || + hasCode(error, "rate_limited") || + status === 429 || + status === 503 + ); +}; + +/** Whether a Sandboxes error has a code, on the error or the step error's cause. */ +export const hasCode = (error: unknown, code: string): boolean => { + const seen = error as + | { code?: string; cause?: { code?: string } } + | undefined; + + return seen?.code === code || seen?.cause?.code === code; +}; + +/** A Sandboxes error's HTTP status, whether on the error or its cause. */ +export const errorStatus = (error: unknown): number | undefined => { + const seen = error as + | { status?: number; cause?: { status?: number } } + | undefined; + + return seen?.status ?? seen?.cause?.status; +}; diff --git a/packages/ci/src/machine/spans.test.ts b/packages/ci/src/machine/spans.test.ts index 681b0df14..b88bbaf4d 100644 --- a/packages/ci/src/machine/spans.test.ts +++ b/packages/ci/src/machine/spans.test.ts @@ -2,148 +2,54 @@ * Tests of the trace spans CI groups its steps under, and what it names them: * one span per job, and in it one per machine, one per command, and one per * attempt for a command with retries, with GitHub check updates in a span of - * their own. Spans and names never change a step's ID. A failed run of a - * command ends in a failing step, so its span shows the failure. CI's own - * work carries CI's origin, and what you wrote carries none. + * their own. Every span has a kind, and a statement on a background process + * is a row of its own. Spans and names never change a step's ID. A failed run + * of a command ends in a failing step, so its span shows the failure. CI's + * own work carries CI's origin, and what you wrote carries none. * * @module */ -import { version as sdkVersion, step } from "inngest"; +import { step } from "inngest"; import { describe, expect, test } from "vitest"; import { CommandFailedError } from "../errors.ts"; -import { consoleReporter } from "../github/auth.ts"; -import { createCi } from "../pipeline/createCi.ts"; import { ciOrigin } from "../pipeline/names.ts"; -import { createCiTestClient } from "../testing/client.ts"; -import { prEvent } from "../testing/events.ts"; -import type { CommandScript, FakeSandboxApi } from "../testing/fakeSandbox.ts"; import { createFakeSandboxApi } from "../testing/fakeSandbox.ts"; -import { runFunction } from "../testing/runFunction.ts"; +import type { Ci } from "../testing/harness.ts"; +import { ciTest, drawTrace } from "../testing/harness.ts"; import { version } from "../version.ts"; import { $ } from "./command.ts"; import { sandbox } from "./sandbox.ts"; -type Ci = ReturnType; - -interface TraceNode { - label: string; - children: Map; -} - -/** - * Draw a run's trace as an indented tree, the way the UI nests it: each span - * once, where its first step is, with its kind in brackets, and each step - * under its spans. - */ -const drawTrace = ( - result: Awaited>, - /** Suffix each row with who it says did it, as in `Create sandbox <- ci`. */ - withOrigins = false, -): string => { - const root: TraceNode = { label: "", children: new Map() }; - - const by = (origin: string | undefined) => { - if (!withOrigins || !origin) { - return ""; - } - - const names: Record = { - [ciOrigin]: "ci", - [`inngest@${sdkVersion}`]: "inngest", - }; - - return ` <- ${names[origin] ?? origin}`; - }; - - for (const stepId of result.stepIds) { - let node = root; - - for (const span of result.spans[stepId] ?? []) { - const key = `span:${span.id}`; - const label = span.kind ? `${span.name} [${span.kind}]` : span.name; - - const child = node.children.get(key) ?? { - label: `${label}${by(span.origin)}`, - children: new Map(), - }; - - node.children.set(key, child); - - node = child; - } - - node.children.set(`step:${stepId}`, { - label: `${result.names[stepId] ?? stepId}${by(result.origins[stepId])}`, - children: new Map(), - }); - } - - const lines: string[] = []; - - const draw = (node: TraceNode, depth: number) => { - for (const child of node.children.values()) { - lines.push(`${" ".repeat(depth)}${child.label}`); - - draw(child, depth + 1); - } - }; - - draw(root, 0); - - return lines.join("\n"); -}; - -/** Run the pipeline whose handler `define` makes from a fresh client. */ -const runPipeline = async ( - define: (ci: Ci) => () => Promise, - scripts: CommandScript[] = [], - api: FakeSandboxApi = createFakeSandboxApi(), -) => { - api.script(scripts); - - const ci = createCi(createCiTestClient(api), { - github: consoleReporter(), - }); - - const pipeline = ci.pipeline( - { id: "pr", on: [{ event: "github/pull_request.opened" }] }, - define(ci), - ); - - return runFunction(pipeline, { event: prEvent }); -}; - /** Run a job with a captured, a managed, a retried and a background command. */ const run = () => { - return runPipeline( - (ci) => { - return ci.job("test", async () => { - await $`quick`.timeout("30s"); + return ciTest({ + scripts: [ + { match: "slow", ticks: 1, stdout: "done" }, + { match: "flaky", exitCode: 1 }, + { match: "serve", ticks: 1, stdout: "listening" }, + ], + }).run((ci) => { + return ci.job("test", async () => { + await $`quick`.timeout("30s"); - await $`slow`; + await $`slow`; - try { - await $`flaky`.retries(1); - } catch { - // Both attempts fail. - } + try { + await $`flaky`.retries(1); + } catch { + // Both attempts fail. + } - const server = await $`serve`.background(); + const server = await $`serve`.background(); - await server.output(); + await server.output(); - await server.exited(); + await server.exited(); - await server.kill(); - }); - }, - [ - { match: "slow", ticks: 1, stdout: "done" }, - { match: "flaky", exitCode: 1 }, - { match: "serve", ticks: 1, stdout: "listening" }, - ], - ); + await server.kill(); + }); + }); }; /** @@ -173,68 +79,73 @@ const rowsPipeline = (ci: Ci) => { }); }; -/** The span of a command `test` ran. */ -const command = (label: string) => { - return { id: `test › ${label}`, name: `$ ${label}` }; -}; - const origin = ciOrigin; -const github = [{ id: "github", name: "GitHub", origin }]; -const job = { id: "test", name: "test", kind: "job" }; -const machine = [job, { id: "test › machine", name: "Start sandbox", origin }]; -const quick = [job, command("quick")]; -const slow = [job, command("slow")]; -const attempt1 = [ - job, - command("flaky"), - { id: "attempt-1", name: "Attempt 1", origin }, -]; -const attempt2 = [ +const span = (id: string, name: string, kind: string) => { + return { id, name, kind }; +}; +const github = [{ ...span("github", "GitHub", "github"), origin }]; +const job = span("test", "test", "job"); +const machine = [ job, - command("flaky"), - { id: "attempt-2", name: "Attempt 2", origin }, + { ...span("test › machine", "Start sandbox", "sandbox"), origin }, ]; -const serve = [job, command("serve")]; +const command = (label: string) => { + return [job, span(`test › ${label}`, `$ ${label}`, "command")]; +}; +const attempt = (n: number) => { + return [ + ...command("flaky"), + { ...span(`attempt-${n}`, `Attempt ${n}`, "attempt"), origin }, + ]; +}; + +/** A statement on the background `serve`, a row of its own beside the command's. */ +const statement = (suffix: string, label: string) => { + return [ + job, + span(`test › serve › ${suffix}`, `$ serve (${label})`, "command"), + ]; +}; describe("spans", () => { - test("group each job, machine, command and retried attempt", async () => { + test("group each job, machine, command and retried attempt, each with its kind", async () => { const result = await run(); expect(result.type).toBe("function-resolved"); - // Background calls re-enter their command's span. Check updates are in - // the run's GitHub span, and cleanup is in none. + // Each statement on a background process is a span of its own. Check + // updates are in the run's GitHub span, and cleanup is in none. expect(result.spans).toEqual({ "github › check:pr:start": github, "github › check:test:start": github, "test › machine": machine, "test › machine › setup": machine, - "test › quick": quick, - "test › slow › start": slow, - "test › slow › wait #1": slow, - "test › slow › check #1": slow, - "test › slow › wait #2": slow, - "test › slow › check #2": slow, - "test › slow › output": slow, - "test › flaky #attempt-1 › start": attempt1, - "test › flaky #attempt-1 › wait #1": attempt1, - "test › flaky #attempt-1 › check #1": attempt1, - "test › flaky #attempt-1 › output": attempt1, - "test › flaky #attempt-1 › exit": attempt1, + "test › quick": command("quick"), + "test › slow › start": command("slow"), + "test › slow › wait #1": command("slow"), + "test › slow › check #1": command("slow"), + "test › slow › wait #2": command("slow"), + "test › slow › check #2": command("slow"), + "test › slow › output": command("slow"), + "test › flaky #attempt-1 › start": attempt(1), + "test › flaky #attempt-1 › wait #1": attempt(1), + "test › flaky #attempt-1 › check #1": attempt(1), + "test › flaky #attempt-1 › output": attempt(1), + "test › flaky #attempt-1 › exit": attempt(1), "github › check:test:attempt:1": github, - "test › flaky #attempt-2 › start": attempt2, - "test › flaky #attempt-2 › wait #1": attempt2, - "test › flaky #attempt-2 › check #1": attempt2, - "test › flaky #attempt-2 › output": attempt2, - "test › flaky #attempt-2 › exit": attempt2, - "test › serve › start": serve, - "test › serve › output #1": serve, - "test › serve › wait #2": serve, - "test › serve › check #2": serve, - "test › serve › wait #3": serve, - "test › serve › check #3": serve, - "test › serve › output": serve, - "test › serve › kill": serve, + "test › flaky #attempt-2 › start": attempt(2), + "test › flaky #attempt-2 › wait #1": attempt(2), + "test › flaky #attempt-2 › check #1": attempt(2), + "test › flaky #attempt-2 › output": attempt(2), + "test › flaky #attempt-2 › exit": attempt(2), + "test › serve › start": command("serve"), + "test › serve › output #1": statement("output #1", "output"), + "test › serve › wait #2": statement("exited #1", "exited"), + "test › serve › check #2": statement("exited #1", "exited"), + "test › serve › wait #3": statement("exited #1", "exited"), + "test › serve › check #3": statement("exited #1", "exited"), + "test › serve › output": statement("exited #1", "exited"), + "test › serve › kill": statement("kill", "kill"), "github › check:test:complete": github, "github › check:jobs:complete": github, "github › check:pr:complete": github, @@ -316,7 +227,7 @@ describe("spans", () => { expect(attempt1Steps.at(-1)).toBe("test › flaky #attempt-1 › exit"); - expect(result.spans["test › flaky #attempt-1 › exit"]).toEqual(attempt1); + expect(result.spans["test › flaky #attempt-1 › exit"]).toEqual(attempt(1)); expect(result.names["test › flaky #attempt-1 › exit"]).toBe( "Exited with code 1", @@ -329,55 +240,41 @@ describe("spans", () => { }); test("record nothing more for a passing or `.nothrow()` command", async () => { - const result = await runPipeline( - (ci) => { - return ci.job("test", async () => { - await $`pass`; - - await $`fail`.nothrow(); - }); - }, - [{ match: "fail", exitCode: 1 }], - ); + const result = await ciTest({ + scripts: [{ match: "fail", exitCode: 1 }], + }).run((ci) => { + return ci.job("test", async () => { + await $`pass`; + + await $`fail`.nothrow(); + }); + }); expect(result.type).toBe("function-resolved"); - expect(result.stepIds).toEqual([ - "github › check:pr:start", - "github › check:test:start", - "test › machine", - "test › machine › setup", - "test › pass › start", - "test › pass › wait #1", - "test › pass › check #1", - "test › pass › output", - "test › fail › start", - "test › fail › wait #1", - "test › fail › check #1", - "test › fail › output", - "github › check:test:complete", - "github › check:jobs:complete", - "github › check:pr:complete", - "pipeline › cleanup", - "pipeline › cleanup:snapshots", - ]); + expect( + result.stepIds.filter((id) => { + return id.endsWith("› exit"); + }), + ).toEqual([]); }); test("still retry after a failed run, and throw the same error at the end", async () => { let caught: unknown; - const result = await runPipeline( - (ci) => { - return ci.job("test", async () => { - try { - await $`flaky`.retries(1); - } catch (error) { - caught = error; - } - }); - }, - [{ match: "flaky", exitCode: 1, stdout: "out", stderr: "flake" }], - ); + const result = await ciTest({ + scripts: [ + { match: "flaky", exitCode: 1, stdout: "out", stderr: "flake" }, + ], + }).run((ci) => { + return ci.job("test", async () => { + try { + await $`flaky`.retries(1); + } catch (error) { + caught = error; + } + }); + }); expect(result.type).toBe("function-resolved"); @@ -396,7 +293,7 @@ describe("spans", () => { }); test("keep the build of a parent outside every job's span", async () => { - const result = await runPipeline((ci) => { + const result = await ciTest().run((ci) => { const base = ci.job("base", async () => { await $`install`; }); @@ -435,13 +332,13 @@ describe("spans", () => { }); test("name a pipeline's rows for what they are", async () => { - const result = await runPipeline(rowsPipeline); + const result = await ciTest().run(rowsPipeline); expect(result.type).toBe("function-resolved"); expect(drawTrace(result)).toBe( [ - "GitHub", + "GitHub [github]", " Create check: pr", " Report test: started", " Report test: passed", @@ -450,25 +347,25 @@ describe("spans", () => { "Look up cache", "Build base in its own run", "test [job]", - " Start sandbox from base", + " Start sandbox from base [sandbox]", " Create sandbox", " Prepare workspace", - " $ pnpm test", + " $ pnpm test [command]", " Start process", " Wait 1s", " Poll process", " Read output", - " lint", + " lint [command]", " Start process", " Wait 1s", " Poll process", " Read output", " my-step", - " api", - " Start sandbox", + " api [sandbox]", + " Start sandbox [sandbox]", " Create sandbox", " Prepare workspace", - " $ pnpm start", + " $ pnpm start [command]", " Run and read output", "Clean up sandboxes", "Clean up snapshots", @@ -477,18 +374,16 @@ describe("spans", () => { }); test("mark the work CI does for you, and nothing you wrote", async () => { - const result = await runPipeline(rowsPipeline); + const result = await ciTest().run(rowsPipeline); expect(result.type).toBe("function-resolved"); expect(ciOrigin).toBe(`@inngest/ci@${version}`); - // Jobs, commands, `my-step` and the `api` sandbox are yours. `snapshot()` - // opens its span without an origin, so it inherits the one of the CI span - // it runs in, while its own steps are marked by the SDK. + // Jobs, commands, `my-step` and the `api` sandbox are yours. expect(drawTrace(result, true)).toBe( [ - "GitHub <- ci", + "GitHub [github] <- ci", " Create check: pr <- ci", " Report test: started <- ci", " Report test: passed <- ci", @@ -497,25 +392,25 @@ describe("spans", () => { "Look up cache <- ci", "Build base in its own run <- ci", "test [job]", - " Start sandbox from base <- ci", + " Start sandbox from base [sandbox] <- ci", " Create sandbox <- ci", " Prepare workspace <- ci", - " $ pnpm test", + " $ pnpm test [command]", " Start process <- ci", " Wait 1s <- ci", " Poll process <- ci", " Read output <- ci", - " lint", + " lint [command]", " Start process <- ci", " Wait 1s <- ci", " Poll process <- ci", " Read output <- ci", " my-step", - " api", - " Start sandbox <- ci", + " api [sandbox]", + " Start sandbox [sandbox] <- ci", " Create sandbox <- ci", " Prepare workspace <- ci", - " $ pnpm start", + " $ pnpm start [command]", " Run and read output <- ci", "Clean up sandboxes <- ci", "Clean up snapshots <- ci", @@ -550,19 +445,15 @@ describe("spans", () => { return fetch(input, init); }; - const result = await runPipeline( - (ci) => { - const parent = ci.job("base", async () => { - await $`pnpm install`; - }); - - return ci.job({ id: "test", from: parent }, async () => { - await $`pnpm test`; - }); - }, - [], - api, - ); + const result = await ciTest({ api }).run((ci) => { + const parent = ci.job("base", async () => { + await $`pnpm install`; + }); + + return ci.job({ id: "test", from: parent }, async () => { + await $`pnpm test`; + }); + }); expect(result.type).toBe("function-resolved"); diff --git a/packages/ci/src/machine/start.ts b/packages/ci/src/machine/start.ts new file mode 100644 index 000000000..5ade4f307 --- /dev/null +++ b/packages/ci/src/machine/start.ts @@ -0,0 +1,335 @@ +/** + * Starting a job's machine: creating it, setting it up, and falling back when + * the snapshot it restores won't start. + * + * @module + */ + +import { snapshotState } from "../cache/cache.ts"; +import { CiUsageError } from "../errors.ts"; +import { traceName } from "../pipeline/names.ts"; +import type { + CiJobScope, + CiRunScope, + MachineHandle, +} from "../pipeline/scope.ts"; +import { defaultCwd, joinId } from "../pipeline/scope.ts"; +import type { MachineConfig } from "../types.ts"; +import { boundedName, errorMessage, slug } from "../util.ts"; +import { isStartFailure } from "./sandboxErrors.ts"; + +/** + * Memory is paired with vCPU count, so a job only picks one number. + */ +const memoryForVcpu = { 1: 1024, 2: 2048, 4: 4096 } as const; + +const resolveMachineConfig = ( + config: MachineConfig | undefined, +): { vcpu: 1 | 2 | 4; memoryMb: number } => { + const vcpu = config?.vcpu ?? 2; + + return { vcpu, memoryMb: memoryForVcpu[vcpu] }; +}; + +/** + * The name a machine is created with. It carries the run ID so orphaned + * machines can be found and destroyed by the cleanup function. + */ +const machineName = (runId: string, path: string): string => { + return boundedName(`ci-${runId}-${slug(path)}`); +}; + +/** + * Run once on every machine before its first command. + * + * WORKAROUNDS (Sandboxes API), delete each part once the platform covers it: + * - Commands run in `/work` by default, and a sandbox won't start a process in + * a directory that doesn't exist, so it's created here. `checkout()` also + * makes it, but a job doesn't have to check out. + * - Sandboxes boot with the loopback interface down, so nothing can listen on + * or reach `127.0.0.1`, which breaks services started with `.background()` + * and `waitForHttp`/`waitForPort`. Bringing it up is a no-op once the + * platform does it itself. + */ +export const machineSetupScript = `mkdir -p ${defaultCwd} && (ip link set lo up 2>/dev/null || true)`; + +/** + * The stages a machine is started in. Only the first is tried every time: + * the rest follow a snapshot that won't start. Each has its own step ID, + * machine name and trace name, all derived from the key. + */ +const stages = { + first: traceName.createMachine, + restart: traceName.restartMachine, + retry: traceName.retryCreate, + fresh: traceName.createFreshMachine, +}; + +type Stage = keyof typeof stages; + +type Sandbox = MachineHandle["sandbox"]; + +/** + * Create this scope's machine, from the snapshot it starts from if it has one. + * + * A snapshot that won't start is retried once, and a second failure makes it + * broken. Either way the job doesn't fail: the parent is rebuilt for it to + * start from, and a fresh machine is the last resort. + */ +export const startMachine = async ( + scope: CiJobScope, + /** The first stage's step ID, which the other stages build on. */ + stepId: string, +): Promise => { + const sandbox = await startSandbox(scope, stepId); + + return { + sandbox, + name: machineName(scope.run.runId, scope.path), + id: sandbox.id, + }; +}; + +const startSandbox = async ( + scope: CiJobScope, + stepId: string, +): Promise => { + const { run } = scope; + const tools = run.sandboxTools; + + if (!tools) { + throw new CiUsageError( + "This pipeline has no `step.sandbox` tools. `createCi` adds `sandboxMiddleware()` to the functions it creates, so this usually means the function was created another way.", + ); + } + + const machineConfig = resolveMachineConfig( + scope.config.machine ?? run.machine ?? run.ci.defaultMachine, + ); + + const idOf = (stage: Stage): string => { + return stage === "first" ? stepId : joinId(stepId, stage); + }; + + const nameOf = (stage: Stage): string => { + return machineName( + run.runId, + stage === "first" ? scope.path : `${scope.path} ${stage}`, + ); + }; + + /** Create a machine at a stage, from a snapshot if one is given, and set it up. */ + const start = async (stage: Stage, snapshotId?: string): Promise => { + const step = { id: idOf(stage), name: stages[stage] }; + const name = nameOf(stage); + + const sandbox = await tools.create( + step, + snapshotId ? { name, snapshotId } : { name, ...machineConfig }, + ); + + await sandbox.commands.run( + { id: joinId(step.id, "setup"), name: traceName.prepareWorkspace }, + ["/bin/sh", "-c", machineSetupScript], + ); + + return sandbox; + }; + + /** + * Start from a snapshot, or the start failure that says it won't. The failed + * start still holds its name, so what follows gets another, and the stuck + * machine is cleared away meanwhile. + */ + const restore = async ( + stage: Stage, + snapshotId: string, + ): Promise<{ started: Sandbox } | { failure: unknown }> => { + try { + return { started: await start(stage, snapshotId) }; + } catch (failure) { + if (!isStartFailure(failure)) { + throw failure; + } + + await discardFailedStart(run, idOf(stage), nameOf(stage), failure); + + return { failure }; + } + }; + + const startFresh = (note: string): Promise => { + scope.fromSnapshotId = undefined; + scope.startNote = note; + scope.restoreFallback = scope.rebuildParent; + + run.ci.hooks.activity(run, scope.jobPath, note); + + return start("fresh"); + }; + + run.ci.hooks.activity( + run, + scope.jobPath, + scope.startNote ?? "creating machine…", + ); + + const snapshotId = scope.fromSnapshotId; + + if (!snapshotId) { + return start("first"); + } + + const parentId = scope.fromJobIds[0] ?? "the parent"; + + // Every job starting from a snapshot checks it for itself, so what a job + // does never depends on whether a sibling got there first. Only the + // rebuild of the parent, which each of them awaits, is shared. + const first = await restore("first", snapshotId); + + if ("started" in first) { + return first.started; + } + + /** Why the snapshot can't be used. */ + let bad = `wouldn't start (${errorMessage(first.failure)})`; + + /** How to replace it. */ + let rebuild = { broken: false, unnamed: false }; + + // WORKAROUND (Sandboxes API): a snapshot can't be told to be broken, so + // it is retried before it is deleted. Delete this once snapshots or + // images report their own health. Today a snapshot has only the + // statuses `CREATING`, `READY`, `DELETING` and `DELETED`, and a broken + // one still says `READY`. A start that times out says no more: the + // machine may have been stuck, or the node busy, as easily as the + // snapshot bad. Deleting on the first one would let a single blip + // destroy a cache that is shared, since a pull request restores its + // base branch's snapshots, and every run after it would build again. So + // only a snapshot that fails to start a second time is broken enough to + // delete. Capacity errors (`compute_unavailable`, 429 and 503) are not + // start failures at all, and fail the job as any other would. + const state = await snapshotState( + run, + joinId(stepId, "snapshot-state"), + snapshotId, + ); + + if (state === "creating") { + // Another build is still taking it and may yet finish, so this + // run rebuilds without contending for its name. + rebuild = { broken: false, unnamed: true }; + } else if (state === "ready") { + // With the default wait, the same as the first try's: a shorter one + // would make a slow node more likely to fail the retry, and a second + // failure deletes the snapshot. + const second = await restore("restart", snapshotId); + + if ("started" in second) { + run.warnings.push( + `retried: snapshot of \`${parentId}\` needed a retry to start (${errorMessage(first.failure)}), so \`${scope.path}\` started it on the second try`, + ); + + return second.started; + } + + bad = `wouldn't start twice (${errorMessage(second.failure)})`; + + rebuild = { broken: true, unnamed: false }; + } + + run.warnings.push( + `fell back: snapshot of \`${parentId}\` ${bad}, so \`${scope.path}\` rebuilt it`, + ); + + const note = `rebuilding ${parentId} · bad snapshot`; + + scope.startNote = note; + + run.ci.hooks.activity(run, scope.jobPath, note); + + // Shared: jobs that had the same trouble wait for one rebuild, which + // also deletes a broken snapshot once. + const replacement = await scope.rebuildSnapshot?.(rebuild); + + if (replacement) { + scope.startNote = `starting ${parentId}`; + + run.ci.hooks.activity(run, scope.jobPath, scope.startNote); + + // Restores may still be broken while fresh machines work, which is where + // this job went before there was a replacement to try. + const retried = await restore("retry", replacement); + + if ("started" in retried) { + return retried.started; + } + } + + return startFresh(note); +}; + +/** + * Best-effort cleanup of a machine that failed to start. The platform keeps + * such a machine (stuck in STARTING) and its name, so it is destroyed here, and + * the end of the run finds it by its name if this couldn't. + * + * Never throws: a machine that can't be cleaned up now must not fail the job. + */ +const discardFailedStart = async ( + run: CiRunScope, + stepId: string, + name: string, + error: unknown, +): Promise => { + const knownId = (error as { cause?: { sandboxId?: string } })?.cause + ?.sandboxId; + + try { + await run.step.run( + { + id: joinId(stepId, "discard"), + name: traceName.discardMachine, + }, + async (): Promise<{ id?: string }> => { + // Errors are swallowed inside the step so it never retries. + let id = knownId; + + try { + let cursor: string | undefined; + + while (!id) { + const page = await run.ci.client.sandboxes.list({ + ...(cursor ? { cursor } : {}), + limit: 100, + }); + + id = page.items.find( + (sandbox: { name: string; status: string }) => { + return sandbox.name === name && sandbox.status !== "TERMINATED"; + }, + )?.id; + + if (id || !page.page.hasMore) { + break; + } + + cursor = page.page.cursor; + } + + if (id) { + const sandbox = await run.ci.client.sandboxes.get(id); + + await sandbox?.destroy(); + } + } catch { + // The run's cleanup finds it again by its name. + } + + return id ? { id } : {}; + }, + ); + } catch { + // Best effort only. + } +}; diff --git a/packages/ci/src/pipeline/cacheGuarantees.test.ts b/packages/ci/src/pipeline/cacheGuarantees.test.ts index c34aa0b88..8814edda2 100644 --- a/packages/ci/src/pipeline/cacheGuarantees.test.ts +++ b/packages/ci/src/pipeline/cacheGuarantees.test.ts @@ -18,7 +18,7 @@ import { describe, expect, test } from "vitest"; import { consoleReporter } from "../github/auth.ts"; import { $ } from "../machine/command.ts"; -import { machineSetupScript } from "../machine/machine.ts"; +import { machineSetupScript } from "../machine/start.ts"; import { createCiTestClient } from "../testing/client.ts"; import { prEvent, prTrigger } from "../testing/events.ts"; import { createFakeSandboxApi } from "../testing/fakeSandbox.ts"; diff --git a/packages/ci/src/pipeline/job.ts b/packages/ci/src/pipeline/job.ts index 447587df6..e482b6d8d 100644 --- a/packages/ci/src/pipeline/job.ts +++ b/packages/ci/src/pipeline/job.ts @@ -38,10 +38,10 @@ import type { CiJobScope, CiRunScope } from "./scope.ts"; import { getRunScope, inJobSpan, + joinId, matrixOriginOf, rootRunIdOf, runJobBody, - scopeSeparator, } from "./scope.ts"; export interface RegisteredJob { @@ -234,10 +234,7 @@ export const invokeBuild = async ({ try { output = (await run.step.invoke( - ciStep( - `${stepPath}${scopeSeparator}build`, - traceName.buildInOwnRun(path), - ), + ciStep(joinId(stepPath, "build"), traceName.buildInOwnRun(path)), { function: run.ci.cacheBuild(), data }, )) as CacheBuildResult | null; } catch (error) { diff --git a/packages/ci/src/pipeline/names.ts b/packages/ci/src/pipeline/names.ts index 82964bf51..475a259b5 100644 --- a/packages/ci/src/pipeline/names.ts +++ b/packages/ci/src/pipeline/names.ts @@ -14,7 +14,8 @@ import type { CheckConclusion } from "../types.ts"; import { formatDuration } from "../util.ts"; import { version } from "../version.ts"; import type { CiRunScope } from "./scope.ts"; -import { scopeSeparator } from "./scope.ts"; +import { joinId, scopeSeparator } from "./scope.ts"; +import type { SpanInfo, SpanKind } from "./spans.ts"; import { originOption } from "./spans.ts"; /** @@ -25,9 +26,72 @@ import { originOption } from "./spans.ts"; */ export const ciOrigin = `@inngest/ci@${version}`; +/** A span for something you wrote or asked for, such as a job or a command. */ +const userSpan = (kind: SpanKind, id: string, name: string): SpanInfo => { + return { kind, id, name }; +}; + /** A span CI opens for its own work, such as starting a sandbox. */ -export const ciSpan = (id: string, name: string) => { - return { id, name, origin: ciOrigin }; +const ciSpan = (kind: SpanKind, id: string, name: string): SpanInfo => { + return { ...userSpan(kind, id, name), origin: ciOrigin }; +}; + +/** + * Every span CI opens: its kind, ID, name and whose work it is, in one place. + * A new span is an entry here and a `inSpan(spans.x(...), fn)` where it + * opens. Spans of what you wrote have no origin. + */ +export const spans = { + /** A job, which holds everything it does. */ + job: (run: CiRunScope, path: string): SpanInfo => { + return userSpan("job", path, traceName.job(run, path)); + }, + + /** The run's one span for reporting to GitHub. */ + github: (): SpanInfo => { + return ciSpan("github", "github", traceName.github); + }, + + /** A machine starting, with its fallbacks. */ + startMachine: (stepId: string, parent?: string): SpanInfo => { + return ciSpan("sandbox", stepId, traceName.startMachine(parent)); + }, + + /** An extra sandbox, named as `sandbox()` was. */ + extraMachine: (path: string, jobPath: string): SpanInfo => { + return userSpan("sandbox", path, traceName.extraMachine(path, jobPath)); + }, + + /** + * A command. A statement on a background process, such as its `kill`, is a + * span of its own beside the command's. + */ + command: ( + stepId: string, + text: string, + label?: string, + statement?: { suffix: string; label: string }, + ): SpanInfo => { + const name = traceName.command(text, label); + + return statement + ? userSpan( + "command", + joinId(stepId, statement.suffix), + `${name} (${statement.label})`, + ) + : userSpan("command", stepId, name); + }, + + /** One attempt of a command that has retries. */ + attempt: (attempt: number): SpanInfo => { + return ciSpan("attempt", `attempt-${attempt}`, traceName.attempt(attempt)); + }, + + /** Saving a job's sandbox: one statement, whatever steps it takes. */ + save: (jobPath: string): SpanInfo => { + return ciSpan("snapshot", joinId(jobPath, "save"), traceName.saveMachine); + }, }; /** @@ -150,6 +214,8 @@ export const traceName = { deleteBadSnapshot: "Delete bad snapshot", checkSnapshotState: "Check snapshot state", + cloneRepository: "Clone repository", + uploadWorkingTree: "Upload working tree", findChangedFiles: "Find changed files", recordStartTime: "Record start time", diff --git a/packages/ci/src/pipeline/pipeline.test.ts b/packages/ci/src/pipeline/pipeline.test.ts index 27d0b17b6..caa4f65fa 100644 --- a/packages/ci/src/pipeline/pipeline.test.ts +++ b/packages/ci/src/pipeline/pipeline.test.ts @@ -22,12 +22,9 @@ import { import { consoleReporter, githubToken } from "../github/auth.ts"; import { github } from "../github/index.ts"; import { $ } from "../machine/command.ts"; -import { - destroyOrphans, - destroyRunMachines, - machineSetupScript, -} from "../machine/machine.ts"; +import { destroyOrphans, destroyRunMachines } from "../machine/machine.ts"; import { sandbox } from "../machine/sandbox.ts"; +import { machineSetupScript } from "../machine/start.ts"; import { report } from "../report.ts"; import { createCiTestClient } from "../testing/client.ts"; import { prEvent, prTrigger } from "../testing/events.ts"; diff --git a/packages/ci/src/pipeline/scope.ts b/packages/ci/src/pipeline/scope.ts index 653f1c2ba..b4f7932b4 100644 --- a/packages/ci/src/pipeline/scope.ts +++ b/packages/ci/src/pipeline/scope.ts @@ -19,7 +19,7 @@ import type { } from "../types.ts"; import type { CacheBuildData, CacheBuildResult } from "./cacheBuild.ts"; import type { CiHooks } from "./hooks.ts"; -import { ciSpan, traceName } from "./names.ts"; +import { spans } from "./names.ts"; import { inSpan } from "./spans.ts"; /** @@ -29,6 +29,11 @@ import { inSpan } from "./spans.ts"; */ export const scopeSeparator = " › "; +/** A step ID built from parts, as in `test › machine › setup`. */ +export const joinId = (...parts: string[]): string => { + return parts.join(scopeSeparator); +}; + /** What a rebuilt job's path adds to its ID, as in `base (rebuild)`. */ export const rebuildSuffix = " (rebuild)"; @@ -416,7 +421,7 @@ export const nextStepId = ( scopePath: string | undefined, label: string, ): string => { - const base = scopePath ? `${scopePath}${scopeSeparator}${label}` : label; + const base = scopePath ? joinId(scopePath, label) : label; const seen = (run.counters.get(base) ?? 0) + 1; run.counters.set(base, seen); @@ -519,7 +524,7 @@ export const withScopePreserved = (tools: T): T => { const prefixStepId = (idOrOptions: unknown, prefix: string): unknown => { if (typeof idOrOptions === "string") { return { - id: `${prefix}${scopeSeparator}${idOrOptions}`, + id: joinId(prefix, idOrOptions), name: idOrOptions, }; } @@ -534,7 +539,7 @@ const prefixStepId = (idOrOptions: unknown, prefix: string): unknown => { return { ...opts, - id: `${prefix}${scopeSeparator}${opts.id}`, + id: joinId(prefix, opts.id), name: opts.name ?? opts.id, }; } @@ -553,10 +558,7 @@ export const inJobSpan = ( fn: () => R, ): R => { return runWithAsyncCtx(run.asyncCtx, () => { - return inSpan( - { id: jobPath, name: traceName.job(run, jobPath), kind: "job" }, - fn, - ); + return inSpan(spans.job(run, jobPath), fn); }); }; @@ -577,7 +579,7 @@ export const outsideJobs = (run: CiRunScope, fn: () => R): R => { */ export const inGitHubSpan = (run: CiRunScope, fn: () => R): R => { return runWithAsyncCtx(run.asyncCtx, () => { - return inSpan(ciSpan("github", traceName.github), fn); + return inSpan(spans.github(), fn); }); }; diff --git a/packages/ci/src/pipeline/spans.test.ts b/packages/ci/src/pipeline/spans.test.ts index 47fd17a5d..100f1de8c 100644 --- a/packages/ci/src/pipeline/spans.test.ts +++ b/packages/ci/src/pipeline/spans.test.ts @@ -1,46 +1,46 @@ /** * Tests of the span helpers, and of CI on an SDK without the span API: with * none, functions still run, step IDs and names are exactly what they are - * with it, and the trace just has no spans or origins. The rest of the suite - * runs with the test stub in place. + * with it, and the trace just has no spans or origins. The same goes for an + * SDK that doesn't take a snapshot's own span. The rest of the suite runs + * with the test stub in place. * * @module */ import { group, step } from "inngest"; -import { afterEach, beforeEach, describe, expect, test } from "vitest"; -import { consoleReporter } from "../github/auth.ts"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { $ } from "../machine/command.ts"; -import { createCiTestClient } from "../testing/client.ts"; -import { prEvent } from "../testing/events.ts"; -import { createFakeSandboxApi } from "../testing/fakeSandbox.ts"; -import { runFunction } from "../testing/runFunction.ts"; +import { snapshotMachine } from "../machine/machine.ts"; +import { ciTest } from "../testing/harness.ts"; import { installSpanStub, removeSpanStub } from "../testing/spanStub.ts"; -import { createCi } from "./createCi.ts"; -import { hasSpanApi, inSpan, originOption } from "./spans.ts"; +import { ciOrigin } from "./names.ts"; +import type { CiJobScope } from "./scope.ts"; +import { + hasSpanApi, + inSnapshotSpan, + inSpan, + originOption, + sdk, + snapshotSpanOption, + versionAtLeast, +} from "./spans.ts"; /** Run a job with a command and a step of your own. */ const run = () => { - const ci = createCi(createCiTestClient(createFakeSandboxApi()), { - github: consoleReporter(), - }); - - const job: () => Promise = ci.job("test", async () => { - await $`pnpm test`; + return ciTest().run((ci) => { + return ci.job("test", async () => { + await $`pnpm test`; - await step.run("mine", () => { - return "done"; + await step.run("mine", () => { + return "done"; + }); }); }); - - const pipeline = ci.pipeline( - { id: "pr", on: [{ event: "github/pull_request.opened" }] }, - job, - ); - - return runFunction(pipeline, { event: prEvent }); }; +const span = { id: "a", name: "A", kind: "job" } as const; + describe("without the SDK's span API", () => { beforeEach(() => { removeSpanStub(); @@ -57,8 +57,6 @@ describe("without the SDK's span API", () => { }); test("inSpan runs the function and returns what it returns", async () => { - const span = { id: "a", name: "A" }; - expect(inSpan(span, () => 7)).toBe(7); await expect( @@ -109,8 +107,8 @@ describe("with the SDK's span API", () => { const calls: unknown[] = []; Object.defineProperty(group, "~span", { - value: (span: unknown, fn: () => unknown) => { - calls.push(span); + value: (opened: unknown, fn: () => unknown) => { + calls.push(opened); return `wrapped ${String(fn())}`; }, @@ -118,13 +116,7 @@ describe("with the SDK's span API", () => { }); try { - const span = { id: "a", name: "A", kind: "job" }; - - expect( - inSpan(span, () => { - return "inner"; - }), - ).toBe("wrapped inner"); + expect(inSpan(span, () => "inner")).toBe("wrapped inner"); expect(calls).toEqual([span]); } finally { @@ -144,3 +136,108 @@ describe("originOption", () => { }); }); }); + +describe("versionAtLeast", () => { + test.each([ + ["4.23.1", true], + ["4.23.2", true], + ["4.24.0", true], + ["5.0.0", true], + ["4.23.1-pr.1", true], + ["4.23.0", false], + ["4.22.9", false], + ["3.99.99", false], + ])("%s", (version, expected) => { + expect(versionAtLeast(version, "4.23.1")).toBe(expected); + }); +}); + +describe("a snapshot's span", () => { + /** What a job's snapshot passes the SDK, and the spans opened around it. */ + const snapshotWith = async (takesSpan: boolean) => { + vi.spyOn(sdk, "takesSnapshotSpan").mockReturnValue(takesSpan); + + const steps: unknown[] = []; + const opened: unknown[] = []; + + const open = vi.fn((spanned: unknown, fn: () => unknown) => { + opened.push(spanned); + + return fn(); + }); + + Object.defineProperty(group, "~span", { + value: open, + configurable: true, + }); + + const scope = { + path: "test", + machine: Promise.resolve({ + sandbox: { + snapshot: (options: unknown) => { + steps.push(options); + + return Promise.resolve({ id: "snap" }); + }, + }, + }), + run: { ci: { hooks: { activity: vi.fn() } } }, + } as unknown as CiJobScope; + + try { + await snapshotMachine(scope); + } finally { + Reflect.deleteProperty(group, "~span"); + + installSpanStub(); + + vi.restoreAllMocks(); + } + + return { steps, opened }; + }; + + const save = { + id: "test › save", + name: "Save sandbox", + kind: "snapshot", + origin: ciOrigin, + }; + + test("goes to an SDK that takes it, as the step's own span", async () => { + removeSpanStub(); + + const { steps, opened } = await snapshotWith(true); + + expect(steps).toEqual([ + expect.objectContaining({ id: "test › snapshot", "~span": save }), + ]); + + expect(opened).toEqual([]); + }); + + test("is opened here for an SDK that doesn't, which would open its own", async () => { + removeSpanStub(); + + const { steps, opened } = await snapshotWith(false); + + expect(steps).toEqual([expect.not.objectContaining({ "~span": save })]); + + expect(opened).toEqual([save]); + }); + + test("is a step option only where the SDK takes it", () => { + vi.spyOn(sdk, "takesSnapshotSpan").mockReturnValue(true); + + expect(snapshotSpanOption(span)).toEqual({ "~span": span }); + + vi.spyOn(sdk, "takesSnapshotSpan").mockReturnValue(false); + + expect(snapshotSpanOption(span)).toEqual({}); + + expect(inSnapshotSpan(span, () => "ran")).toBe("ran"); + + vi.restoreAllMocks(); + }); +}); diff --git a/packages/ci/src/pipeline/spans.ts b/packages/ci/src/pipeline/spans.ts index 59872b69f..b2471c28e 100644 --- a/packages/ci/src/pipeline/spans.ts +++ b/packages/ci/src/pipeline/spans.ts @@ -10,7 +10,16 @@ * @module */ -import { group } from "inngest"; +import { group, version as sdkVersion } from "inngest"; + +/** What a span stands for. Every span CI opens has one. */ +export type SpanKind = + | "job" + | "sandbox" + | "command" + | "snapshot" + | "github" + | "attempt"; /** A trace span, as the SDK's span API takes it. */ export interface SpanInfo { @@ -18,8 +27,8 @@ export interface SpanInfo { id: string; /** What the span is called in the trace. */ name: string; - /** What kind of span it is, as in `job`. */ - kind?: string; + /** What kind of span it is. */ + kind: SpanKind; /** Who did the work in it, inherited by the steps and spans inside. */ origin?: string; } @@ -68,3 +77,49 @@ export const originOption = ( ): { "~origin": string } => { return { "~origin": origin }; }; + +/** The first SDK whose snapshot takes a caller's `"~span"` as its own span. */ +const snapshotSpanSince = "4.23.1"; + +/** Whether `version` is `since` or later, as in `4.23.1`. */ +export const versionAtLeast = (version: string, since: string): boolean => { + const have = version.split(/[.-]/).map(Number); + + for (const [index, want] of since.split(".").map(Number).entries()) { + const got = have[index] ?? 0; + + if (got !== want) { + return got > want; + } + } + + return true; +}; + +/** + * What the SDK in use can do beyond the span API. Older SDKs ignore the step + * options they don't know, so each is detected here rather than assumed. + */ +export const sdk = { + /** + * Whether a snapshot step's `"~span"` option stands in for the span the SDK + * would open around its create and wait. Older SDKs ignore the option and + * open their own, which then sits inside the span CI opens. + */ + takesSnapshotSpan: (): boolean => { + return versionAtLeast(sdkVersion, snapshotSpanSince); + }, +}; + +/** + * The step option that makes a snapshot one statement in `span`, where the + * SDK takes one. Elsewhere there is none, and `inSnapshotSpan` opens it. + */ +export const snapshotSpanOption = (span: SpanInfo): { "~span"?: SpanInfo } => { + return sdk.takesSnapshotSpan() ? { "~span": span } : {}; +}; + +/** Run `fn`, which takes snapshots, in `span` unless the SDK does it itself. */ +export const inSnapshotSpan = (span: SpanInfo, fn: () => R): R => { + return sdk.takesSnapshotSpan() ? fn() : inSpan(span, fn); +}; diff --git a/packages/ci/src/testing/README.md b/packages/ci/src/testing/README.md index 7893f48ba..bcbca1697 100644 --- a/packages/ci/src/testing/README.md +++ b/packages/ci/src/testing/README.md @@ -7,5 +7,6 @@ Test-only helpers. Not exported and not built. - `client.ts`: an Inngest client wired to the fake sandbox API, which also records the functions it creates. - `events.ts`: the pull request event and trigger tests share. - `spanStub.ts`, `setup.ts`: a stand-in for the SDK's span API, installed before every test file when the SDK has none, so span and origin assertions hold on any SDK. +- `harness.ts`: `ciTest`, a client on the fake API with a `run` for a pipeline a test defines, and `drawTrace` for the trace it left. - `runFunction.ts`: drives a function to completion like the executor, including `step.invoke` of the functions the test client created. Invokes run beside the function, and the function is called again as each one ends; runs of one `event.data` concurrency key take turns. - `schema.ts`: a fake Standard Schema that can also write itself as JSON Schema. diff --git a/packages/ci/src/testing/harness.ts b/packages/ci/src/testing/harness.ts new file mode 100644 index 000000000..4fa5142c2 --- /dev/null +++ b/packages/ci/src/testing/harness.ts @@ -0,0 +1,115 @@ +/** + * What a test of a pipeline starts from: a client on the fake Sandboxes API + * and a `ci` that reports to the console, with `run` driving one pipeline + * whose handler a test defines. `drawTrace` draws the trace such a run left. + * + * @module + */ + +import { version as sdkVersion } from "inngest"; +import { consoleReporter } from "../github/auth.ts"; +import { createCi } from "../pipeline/createCi.ts"; +import { ciOrigin } from "../pipeline/names.ts"; +import { createCiTestClient } from "./client.ts"; +import { prEvent, prTrigger } from "./events.ts"; +import type { CommandScript, FakeSandboxApi } from "./fakeSandbox.ts"; +import { createFakeSandboxApi } from "./fakeSandbox.ts"; +import { runFunction } from "./runFunction.ts"; + +export type Ci = ReturnType; + +export type RunResult = Awaited>; + +export const ciTest = ( + options: { + /** The fake API to run on. Defaults to a fresh one. */ + api?: FakeSandboxApi; + /** What the commands it runs do. */ + scripts?: CommandScript[]; + } = {}, +) => { + const api = options.api ?? createFakeSandboxApi(); + + api.script(options.scripts ?? []); + + const ci = createCi(createCiTestClient(api), { github: consoleReporter() }); + + return { + api, + ci, + /** Run the pipeline whose handler `define` makes from `ci`. */ + run: (define: (ci: Ci) => () => Promise): Promise => { + const pipeline = ci.pipeline({ id: "pr", on: prTrigger }, define(ci)); + + return runFunction(pipeline, { event: prEvent }); + }, + }; +}; + +interface TraceNode { + label: string; + children: Map; +} + +/** + * Draw a run's trace as an indented tree, the way the UI nests it: each span + * once, where its first step is, with its kind in brackets, and each step + * under its spans. + */ +export const drawTrace = ( + result: RunResult, + /** Suffix each row with who it says did it, as in `Create sandbox <- ci`. */ + withOrigins = false, +): string => { + const root: TraceNode = { label: "", children: new Map() }; + + const by = (origin: string | undefined) => { + if (!withOrigins || !origin) { + return ""; + } + + const names: Record = { + [ciOrigin]: "ci", + [`inngest@${sdkVersion}`]: "inngest", + }; + + return ` <- ${names[origin] ?? origin}`; + }; + + for (const stepId of result.stepIds) { + let node = root; + + for (const span of result.spans[stepId] ?? []) { + const key = `span:${span.id}`; + const label = span.kind ? `${span.name} [${span.kind}]` : span.name; + + const child = node.children.get(key) ?? { + label: `${label}${by(span.origin)}`, + children: new Map(), + }; + + node.children.set(key, child); + + node = child; + } + + node.children.set(`step:${stepId}`, { + label: `${result.names[stepId] ?? stepId}${by(result.origins[stepId])}`, + children: new Map(), + }); + } + + const lines: string[] = []; + + const draw = (node: TraceNode, depth: number) => { + for (const child of node.children.values()) { + lines.push(`${" ".repeat(depth)}${child.label}`); + + draw(child, depth + 1); + } + }; + + draw(root, 0); + + return lines.join("\n"); +}; From 5c9fd49cb15b08569ab75eaeba41009e080d1086 Mon Sep 17 00:00:00 2001 From: x Date: Sat, 10 Oct 2026 00:42:13 +0000 Subject: [PATCH 03/11] refactor(ci): keep the staged machine start in machine.ts Moving it to its own file cost more diff than it saved. Co-Authored-By: Claude Opus 5.5 --- packages/ci/src/machine/machine.ts | 383 +++++++++++++++++- packages/ci/src/machine/sandboxErrors.ts | 67 --- packages/ci/src/machine/start.ts | 335 --------------- .../ci/src/pipeline/cacheGuarantees.test.ts | 2 +- packages/ci/src/pipeline/pipeline.test.ts | 7 +- 5 files changed, 386 insertions(+), 408 deletions(-) delete mode 100644 packages/ci/src/machine/sandboxErrors.ts delete mode 100644 packages/ci/src/machine/start.ts diff --git a/packages/ci/src/machine/machine.ts b/packages/ci/src/machine/machine.ts index 2a38b7a1e..8d9751a38 100644 --- a/packages/ci/src/machine/machine.ts +++ b/packages/ci/src/machine/machine.ts @@ -7,7 +7,8 @@ import type { Inngest } from "inngest"; import type { CachedSnapshot, CacheTarget } from "../cache/cache.ts"; -import { resolveTakenName } from "../cache/cache.ts"; +import { resolveTakenName, snapshotState } from "../cache/cache.ts"; +import { CiUsageError } from "../errors.ts"; import { ciStep, spans, traceName } from "../pipeline/names.ts"; import type { CiJobScope, @@ -15,6 +16,7 @@ import type { MachineHandle, } from "../pipeline/scope.ts"; import { + defaultCwd, inJobSpan, isRestoring, joinId, @@ -26,13 +28,14 @@ import { inSpan, snapshotSpanOption, } from "../pipeline/spans.ts"; +import type { MachineConfig } from "../types.ts"; import { + boundedName, errorMessage, isSandboxNotFound, isSnapshotNotFound, + slug, } from "../util.ts"; -import { errorStatus, hasCode } from "./sandboxErrors.ts"; -import { startMachine } from "./start.ts"; /** * Create this scope's machine if it doesn't have one yet. @@ -92,6 +95,380 @@ export const inMachineSpan = (scope: CiJobScope, fn: () => R): R => { return inSpan(spans.extraMachine(scope.path, scope.jobPath), fn); }; +/** + * Memory is paired with vCPU count, so a job only picks one number. + */ +const memoryForVcpu = { 1: 1024, 2: 2048, 4: 4096 } as const; + +const resolveMachineConfig = ( + config: MachineConfig | undefined, +): { vcpu: 1 | 2 | 4; memoryMb: number } => { + const vcpu = config?.vcpu ?? 2; + + return { vcpu, memoryMb: memoryForVcpu[vcpu] }; +}; + +/** + * The name a machine is created with. It carries the run ID so orphaned + * machines can be found and destroyed by the cleanup function. + */ +const machineName = (runId: string, path: string): string => { + return boundedName(`ci-${runId}-${slug(path)}`); +}; + +/** + * Run once on every machine before its first command. + * + * WORKAROUNDS (Sandboxes API), delete each part once the platform covers it: + * - Commands run in `/work` by default, and a sandbox won't start a process in + * a directory that doesn't exist, so it's created here. `checkout()` also + * makes it, but a job doesn't have to check out. + * - Sandboxes boot with the loopback interface down, so nothing can listen on + * or reach `127.0.0.1`, which breaks services started with `.background()` + * and `waitForHttp`/`waitForPort`. Bringing it up is a no-op once the + * platform does it itself. + */ +export const machineSetupScript = `mkdir -p ${defaultCwd} && (ip link set lo up 2>/dev/null || true)`; + +/** + * The stages a machine is started in. Only the first is tried every time: + * the rest follow a snapshot that won't start. Each has its own step ID, + * machine name and trace name, all derived from the key. + */ +const stages = { + first: traceName.createMachine, + restart: traceName.restartMachine, + retry: traceName.retryCreate, + fresh: traceName.createFreshMachine, +}; + +type Stage = keyof typeof stages; + +type Sandbox = MachineHandle["sandbox"]; + +/** + * Create this scope's machine, from the snapshot it starts from if it has one. + * + * A snapshot that won't start is retried once, and a second failure makes it + * broken. Either way the job doesn't fail: the parent is rebuilt for it to + * start from, and a fresh machine is the last resort. + */ +export const startMachine = async ( + scope: CiJobScope, + /** The first stage's step ID, which the other stages build on. */ + stepId: string, +): Promise => { + const sandbox = await startSandbox(scope, stepId); + + return { + sandbox, + name: machineName(scope.run.runId, scope.path), + id: sandbox.id, + }; +}; + +const startSandbox = async ( + scope: CiJobScope, + stepId: string, +): Promise => { + const { run } = scope; + const tools = run.sandboxTools; + + if (!tools) { + throw new CiUsageError( + "This pipeline has no `step.sandbox` tools. `createCi` adds `sandboxMiddleware()` to the functions it creates, so this usually means the function was created another way.", + ); + } + + const machineConfig = resolveMachineConfig( + scope.config.machine ?? run.machine ?? run.ci.defaultMachine, + ); + + const idOf = (stage: Stage): string => { + return stage === "first" ? stepId : joinId(stepId, stage); + }; + + const nameOf = (stage: Stage): string => { + return machineName( + run.runId, + stage === "first" ? scope.path : `${scope.path} ${stage}`, + ); + }; + + /** Create a machine at a stage, from a snapshot if one is given, and set it up. */ + const start = async (stage: Stage, snapshotId?: string): Promise => { + const step = { id: idOf(stage), name: stages[stage] }; + const name = nameOf(stage); + + const sandbox = await tools.create( + step, + snapshotId ? { name, snapshotId } : { name, ...machineConfig }, + ); + + await sandbox.commands.run( + { id: joinId(step.id, "setup"), name: traceName.prepareWorkspace }, + ["/bin/sh", "-c", machineSetupScript], + ); + + return sandbox; + }; + + /** + * Start from a snapshot, or the start failure that says it won't. The failed + * start still holds its name, so what follows gets another, and the stuck + * machine is cleared away meanwhile. + */ + const restore = async ( + stage: Stage, + snapshotId: string, + ): Promise<{ started: Sandbox } | { failure: unknown }> => { + try { + return { started: await start(stage, snapshotId) }; + } catch (failure) { + if (!isStartFailure(failure)) { + throw failure; + } + + await discardFailedStart(run, idOf(stage), nameOf(stage), failure); + + return { failure }; + } + }; + + const startFresh = (note: string): Promise => { + scope.fromSnapshotId = undefined; + scope.startNote = note; + scope.restoreFallback = scope.rebuildParent; + + run.ci.hooks.activity(run, scope.jobPath, note); + + return start("fresh"); + }; + + run.ci.hooks.activity( + run, + scope.jobPath, + scope.startNote ?? "creating machine…", + ); + + const snapshotId = scope.fromSnapshotId; + + if (!snapshotId) { + return start("first"); + } + + const parentId = scope.fromJobIds[0] ?? "the parent"; + + // Every job starting from a snapshot checks it for itself, so what a job + // does never depends on whether a sibling got there first. Only the + // rebuild of the parent, which each of them awaits, is shared. + const first = await restore("first", snapshotId); + + if ("started" in first) { + return first.started; + } + + /** Why the snapshot can't be used. */ + let bad = `wouldn't start (${errorMessage(first.failure)})`; + + /** How to replace it. */ + let rebuild = { broken: false, unnamed: false }; + + // WORKAROUND (Sandboxes API): a snapshot can't be told to be broken, so + // it is retried before it is deleted. Delete this once snapshots or + // images report their own health. Today a snapshot has only the + // statuses `CREATING`, `READY`, `DELETING` and `DELETED`, and a broken + // one still says `READY`. A start that times out says no more: the + // machine may have been stuck, or the node busy, as easily as the + // snapshot bad. Deleting on the first one would let a single blip + // destroy a cache that is shared, since a pull request restores its + // base branch's snapshots, and every run after it would build again. So + // only a snapshot that fails to start a second time is broken enough to + // delete. Capacity errors (`compute_unavailable`, 429 and 503) are not + // start failures at all, and fail the job as any other would. + const state = await snapshotState( + run, + joinId(stepId, "snapshot-state"), + snapshotId, + ); + + if (state === "creating") { + // Another build is still taking it and may yet finish, so this + // run rebuilds without contending for its name. + rebuild = { broken: false, unnamed: true }; + } else if (state === "ready") { + // With the default wait, the same as the first try's: a shorter one + // would make a slow node more likely to fail the retry, and a second + // failure deletes the snapshot. + const second = await restore("restart", snapshotId); + + if ("started" in second) { + run.warnings.push( + `retried: snapshot of \`${parentId}\` needed a retry to start (${errorMessage(first.failure)}), so \`${scope.path}\` started it on the second try`, + ); + + return second.started; + } + + bad = `wouldn't start twice (${errorMessage(second.failure)})`; + + rebuild = { broken: true, unnamed: false }; + } + + run.warnings.push( + `fell back: snapshot of \`${parentId}\` ${bad}, so \`${scope.path}\` rebuilt it`, + ); + + const note = `rebuilding ${parentId} · bad snapshot`; + + scope.startNote = note; + + run.ci.hooks.activity(run, scope.jobPath, note); + + // Shared: jobs that had the same trouble wait for one rebuild, which + // also deletes a broken snapshot once. + const replacement = await scope.rebuildSnapshot?.(rebuild); + + if (replacement) { + scope.startNote = `starting ${parentId}`; + + run.ci.hooks.activity(run, scope.jobPath, scope.startNote); + + // Restores may still be broken while fresh machines work, which is where + // this job went before there was a replacement to try. + const retried = await restore("retry", replacement); + + if ("started" in retried) { + return retried.started; + } + } + + return startFresh(note); +}; + +/** + * Best-effort cleanup of a machine that failed to start. The platform keeps + * such a machine (stuck in STARTING) and its name, so it is destroyed here, and + * the end of the run finds it by its name if this couldn't. + * + * Never throws: a machine that can't be cleaned up now must not fail the job. + */ +const discardFailedStart = async ( + run: CiRunScope, + stepId: string, + name: string, + error: unknown, +): Promise => { + const knownId = (error as { cause?: { sandboxId?: string } })?.cause + ?.sandboxId; + + try { + await run.step.run( + { + id: joinId(stepId, "discard"), + name: traceName.discardMachine, + }, + async (): Promise<{ id?: string }> => { + // Errors are swallowed inside the step so it never retries. + let id = knownId; + + try { + let cursor: string | undefined; + + while (!id) { + const page = await run.ci.client.sandboxes.list({ + ...(cursor ? { cursor } : {}), + limit: 100, + }); + + id = page.items.find( + (sandbox: { name: string; status: string }) => { + return sandbox.name === name && sandbox.status !== "TERMINATED"; + }, + )?.id; + + if (id || !page.page.hasMore) { + break; + } + + cursor = page.page.cursor; + } + + if (id) { + const sandbox = await run.ci.client.sandboxes.get(id); + + await sandbox?.destroy(); + } + } catch { + // The run's cleanup finds it again by its name. + } + + return id ? { id } : {}; + }, + ); + } catch { + // Best effort only. + } +}; + +/** + * Whether a machine failed to start, as opposed to a request that was refused. + * Only a snapshot restore falls back on it: a plain machine that won't start + * fails the job with its reason. + */ +export const isStartFailure = (error: unknown): boolean => { + const codes = ["sandbox_start_timed_out", "sandbox_start_failed"]; + + // No capacity, or too many requests, says nothing about the snapshot. + if (isRetryable(error)) { + return false; + } + + return ( + codes.some((code) => { + return hasCode(error, code); + }) || /did not reach RUNNING/i.test(errorMessage(error)) + ); +}; + +/** + * Whether the platform says to try again, as it does for no capacity + * (`compute_unavailable`) and rate limits (429 and 503). + */ +const isRetryable = (error: unknown): boolean => { + const seen = error as + | { retryable?: boolean; cause?: { retryable?: boolean } } + | undefined; + const status = errorStatus(error); + + return ( + seen?.retryable === true || + seen?.cause?.retryable === true || + hasCode(error, "compute_unavailable") || + hasCode(error, "rate_limited") || + status === 429 || + status === 503 + ); +}; + +/** Whether a Sandboxes error has a code, on the error or the step error's cause. */ +export const hasCode = (error: unknown, code: string): boolean => { + const seen = error as + | { code?: string; cause?: { code?: string } } + | undefined; + + return seen?.code === code || seen?.cause?.code === code; +}; + +/** A Sandboxes error's HTTP status, whether on the error or its cause. */ +export const errorStatus = (error: unknown): number | undefined => { + const seen = error as + | { status?: number; cause?: { status?: number } } + | undefined; + + return seen?.status ?? seen?.cause?.status; +}; + /** How a job's snapshot is named. */ export interface SnapshotCache { target: CacheTarget; diff --git a/packages/ci/src/machine/sandboxErrors.ts b/packages/ci/src/machine/sandboxErrors.ts deleted file mode 100644 index 7b10821f8..000000000 --- a/packages/ci/src/machine/sandboxErrors.ts +++ /dev/null @@ -1,67 +0,0 @@ -/** - * What a Sandboxes error says about itself: whether a machine failed to - * start, whether the platform says to try again, and the code and status it - * carries, on the error or on the step error's cause. - * - * @module - */ - -import { errorMessage } from "../util.ts"; - -/** - * Whether a machine failed to start, as opposed to a request that was refused. - * Only a snapshot restore falls back on it: a plain machine that won't start - * fails the job with its reason. - */ -export const isStartFailure = (error: unknown): boolean => { - const codes = ["sandbox_start_timed_out", "sandbox_start_failed"]; - - // No capacity, or too many requests, says nothing about the snapshot. - if (isRetryable(error)) { - return false; - } - - return ( - codes.some((code) => { - return hasCode(error, code); - }) || /did not reach RUNNING/i.test(errorMessage(error)) - ); -}; - -/** - * Whether the platform says to try again, as it does for no capacity - * (`compute_unavailable`) and rate limits (429 and 503). - */ -const isRetryable = (error: unknown): boolean => { - const seen = error as - | { retryable?: boolean; cause?: { retryable?: boolean } } - | undefined; - const status = errorStatus(error); - - return ( - seen?.retryable === true || - seen?.cause?.retryable === true || - hasCode(error, "compute_unavailable") || - hasCode(error, "rate_limited") || - status === 429 || - status === 503 - ); -}; - -/** Whether a Sandboxes error has a code, on the error or the step error's cause. */ -export const hasCode = (error: unknown, code: string): boolean => { - const seen = error as - | { code?: string; cause?: { code?: string } } - | undefined; - - return seen?.code === code || seen?.cause?.code === code; -}; - -/** A Sandboxes error's HTTP status, whether on the error or its cause. */ -export const errorStatus = (error: unknown): number | undefined => { - const seen = error as - | { status?: number; cause?: { status?: number } } - | undefined; - - return seen?.status ?? seen?.cause?.status; -}; diff --git a/packages/ci/src/machine/start.ts b/packages/ci/src/machine/start.ts deleted file mode 100644 index 5ade4f307..000000000 --- a/packages/ci/src/machine/start.ts +++ /dev/null @@ -1,335 +0,0 @@ -/** - * Starting a job's machine: creating it, setting it up, and falling back when - * the snapshot it restores won't start. - * - * @module - */ - -import { snapshotState } from "../cache/cache.ts"; -import { CiUsageError } from "../errors.ts"; -import { traceName } from "../pipeline/names.ts"; -import type { - CiJobScope, - CiRunScope, - MachineHandle, -} from "../pipeline/scope.ts"; -import { defaultCwd, joinId } from "../pipeline/scope.ts"; -import type { MachineConfig } from "../types.ts"; -import { boundedName, errorMessage, slug } from "../util.ts"; -import { isStartFailure } from "./sandboxErrors.ts"; - -/** - * Memory is paired with vCPU count, so a job only picks one number. - */ -const memoryForVcpu = { 1: 1024, 2: 2048, 4: 4096 } as const; - -const resolveMachineConfig = ( - config: MachineConfig | undefined, -): { vcpu: 1 | 2 | 4; memoryMb: number } => { - const vcpu = config?.vcpu ?? 2; - - return { vcpu, memoryMb: memoryForVcpu[vcpu] }; -}; - -/** - * The name a machine is created with. It carries the run ID so orphaned - * machines can be found and destroyed by the cleanup function. - */ -const machineName = (runId: string, path: string): string => { - return boundedName(`ci-${runId}-${slug(path)}`); -}; - -/** - * Run once on every machine before its first command. - * - * WORKAROUNDS (Sandboxes API), delete each part once the platform covers it: - * - Commands run in `/work` by default, and a sandbox won't start a process in - * a directory that doesn't exist, so it's created here. `checkout()` also - * makes it, but a job doesn't have to check out. - * - Sandboxes boot with the loopback interface down, so nothing can listen on - * or reach `127.0.0.1`, which breaks services started with `.background()` - * and `waitForHttp`/`waitForPort`. Bringing it up is a no-op once the - * platform does it itself. - */ -export const machineSetupScript = `mkdir -p ${defaultCwd} && (ip link set lo up 2>/dev/null || true)`; - -/** - * The stages a machine is started in. Only the first is tried every time: - * the rest follow a snapshot that won't start. Each has its own step ID, - * machine name and trace name, all derived from the key. - */ -const stages = { - first: traceName.createMachine, - restart: traceName.restartMachine, - retry: traceName.retryCreate, - fresh: traceName.createFreshMachine, -}; - -type Stage = keyof typeof stages; - -type Sandbox = MachineHandle["sandbox"]; - -/** - * Create this scope's machine, from the snapshot it starts from if it has one. - * - * A snapshot that won't start is retried once, and a second failure makes it - * broken. Either way the job doesn't fail: the parent is rebuilt for it to - * start from, and a fresh machine is the last resort. - */ -export const startMachine = async ( - scope: CiJobScope, - /** The first stage's step ID, which the other stages build on. */ - stepId: string, -): Promise => { - const sandbox = await startSandbox(scope, stepId); - - return { - sandbox, - name: machineName(scope.run.runId, scope.path), - id: sandbox.id, - }; -}; - -const startSandbox = async ( - scope: CiJobScope, - stepId: string, -): Promise => { - const { run } = scope; - const tools = run.sandboxTools; - - if (!tools) { - throw new CiUsageError( - "This pipeline has no `step.sandbox` tools. `createCi` adds `sandboxMiddleware()` to the functions it creates, so this usually means the function was created another way.", - ); - } - - const machineConfig = resolveMachineConfig( - scope.config.machine ?? run.machine ?? run.ci.defaultMachine, - ); - - const idOf = (stage: Stage): string => { - return stage === "first" ? stepId : joinId(stepId, stage); - }; - - const nameOf = (stage: Stage): string => { - return machineName( - run.runId, - stage === "first" ? scope.path : `${scope.path} ${stage}`, - ); - }; - - /** Create a machine at a stage, from a snapshot if one is given, and set it up. */ - const start = async (stage: Stage, snapshotId?: string): Promise => { - const step = { id: idOf(stage), name: stages[stage] }; - const name = nameOf(stage); - - const sandbox = await tools.create( - step, - snapshotId ? { name, snapshotId } : { name, ...machineConfig }, - ); - - await sandbox.commands.run( - { id: joinId(step.id, "setup"), name: traceName.prepareWorkspace }, - ["/bin/sh", "-c", machineSetupScript], - ); - - return sandbox; - }; - - /** - * Start from a snapshot, or the start failure that says it won't. The failed - * start still holds its name, so what follows gets another, and the stuck - * machine is cleared away meanwhile. - */ - const restore = async ( - stage: Stage, - snapshotId: string, - ): Promise<{ started: Sandbox } | { failure: unknown }> => { - try { - return { started: await start(stage, snapshotId) }; - } catch (failure) { - if (!isStartFailure(failure)) { - throw failure; - } - - await discardFailedStart(run, idOf(stage), nameOf(stage), failure); - - return { failure }; - } - }; - - const startFresh = (note: string): Promise => { - scope.fromSnapshotId = undefined; - scope.startNote = note; - scope.restoreFallback = scope.rebuildParent; - - run.ci.hooks.activity(run, scope.jobPath, note); - - return start("fresh"); - }; - - run.ci.hooks.activity( - run, - scope.jobPath, - scope.startNote ?? "creating machine…", - ); - - const snapshotId = scope.fromSnapshotId; - - if (!snapshotId) { - return start("first"); - } - - const parentId = scope.fromJobIds[0] ?? "the parent"; - - // Every job starting from a snapshot checks it for itself, so what a job - // does never depends on whether a sibling got there first. Only the - // rebuild of the parent, which each of them awaits, is shared. - const first = await restore("first", snapshotId); - - if ("started" in first) { - return first.started; - } - - /** Why the snapshot can't be used. */ - let bad = `wouldn't start (${errorMessage(first.failure)})`; - - /** How to replace it. */ - let rebuild = { broken: false, unnamed: false }; - - // WORKAROUND (Sandboxes API): a snapshot can't be told to be broken, so - // it is retried before it is deleted. Delete this once snapshots or - // images report their own health. Today a snapshot has only the - // statuses `CREATING`, `READY`, `DELETING` and `DELETED`, and a broken - // one still says `READY`. A start that times out says no more: the - // machine may have been stuck, or the node busy, as easily as the - // snapshot bad. Deleting on the first one would let a single blip - // destroy a cache that is shared, since a pull request restores its - // base branch's snapshots, and every run after it would build again. So - // only a snapshot that fails to start a second time is broken enough to - // delete. Capacity errors (`compute_unavailable`, 429 and 503) are not - // start failures at all, and fail the job as any other would. - const state = await snapshotState( - run, - joinId(stepId, "snapshot-state"), - snapshotId, - ); - - if (state === "creating") { - // Another build is still taking it and may yet finish, so this - // run rebuilds without contending for its name. - rebuild = { broken: false, unnamed: true }; - } else if (state === "ready") { - // With the default wait, the same as the first try's: a shorter one - // would make a slow node more likely to fail the retry, and a second - // failure deletes the snapshot. - const second = await restore("restart", snapshotId); - - if ("started" in second) { - run.warnings.push( - `retried: snapshot of \`${parentId}\` needed a retry to start (${errorMessage(first.failure)}), so \`${scope.path}\` started it on the second try`, - ); - - return second.started; - } - - bad = `wouldn't start twice (${errorMessage(second.failure)})`; - - rebuild = { broken: true, unnamed: false }; - } - - run.warnings.push( - `fell back: snapshot of \`${parentId}\` ${bad}, so \`${scope.path}\` rebuilt it`, - ); - - const note = `rebuilding ${parentId} · bad snapshot`; - - scope.startNote = note; - - run.ci.hooks.activity(run, scope.jobPath, note); - - // Shared: jobs that had the same trouble wait for one rebuild, which - // also deletes a broken snapshot once. - const replacement = await scope.rebuildSnapshot?.(rebuild); - - if (replacement) { - scope.startNote = `starting ${parentId}`; - - run.ci.hooks.activity(run, scope.jobPath, scope.startNote); - - // Restores may still be broken while fresh machines work, which is where - // this job went before there was a replacement to try. - const retried = await restore("retry", replacement); - - if ("started" in retried) { - return retried.started; - } - } - - return startFresh(note); -}; - -/** - * Best-effort cleanup of a machine that failed to start. The platform keeps - * such a machine (stuck in STARTING) and its name, so it is destroyed here, and - * the end of the run finds it by its name if this couldn't. - * - * Never throws: a machine that can't be cleaned up now must not fail the job. - */ -const discardFailedStart = async ( - run: CiRunScope, - stepId: string, - name: string, - error: unknown, -): Promise => { - const knownId = (error as { cause?: { sandboxId?: string } })?.cause - ?.sandboxId; - - try { - await run.step.run( - { - id: joinId(stepId, "discard"), - name: traceName.discardMachine, - }, - async (): Promise<{ id?: string }> => { - // Errors are swallowed inside the step so it never retries. - let id = knownId; - - try { - let cursor: string | undefined; - - while (!id) { - const page = await run.ci.client.sandboxes.list({ - ...(cursor ? { cursor } : {}), - limit: 100, - }); - - id = page.items.find( - (sandbox: { name: string; status: string }) => { - return sandbox.name === name && sandbox.status !== "TERMINATED"; - }, - )?.id; - - if (id || !page.page.hasMore) { - break; - } - - cursor = page.page.cursor; - } - - if (id) { - const sandbox = await run.ci.client.sandboxes.get(id); - - await sandbox?.destroy(); - } - } catch { - // The run's cleanup finds it again by its name. - } - - return id ? { id } : {}; - }, - ); - } catch { - // Best effort only. - } -}; diff --git a/packages/ci/src/pipeline/cacheGuarantees.test.ts b/packages/ci/src/pipeline/cacheGuarantees.test.ts index 8814edda2..c34aa0b88 100644 --- a/packages/ci/src/pipeline/cacheGuarantees.test.ts +++ b/packages/ci/src/pipeline/cacheGuarantees.test.ts @@ -18,7 +18,7 @@ import { describe, expect, test } from "vitest"; import { consoleReporter } from "../github/auth.ts"; import { $ } from "../machine/command.ts"; -import { machineSetupScript } from "../machine/start.ts"; +import { machineSetupScript } from "../machine/machine.ts"; import { createCiTestClient } from "../testing/client.ts"; import { prEvent, prTrigger } from "../testing/events.ts"; import { createFakeSandboxApi } from "../testing/fakeSandbox.ts"; diff --git a/packages/ci/src/pipeline/pipeline.test.ts b/packages/ci/src/pipeline/pipeline.test.ts index caa4f65fa..27d0b17b6 100644 --- a/packages/ci/src/pipeline/pipeline.test.ts +++ b/packages/ci/src/pipeline/pipeline.test.ts @@ -22,9 +22,12 @@ import { import { consoleReporter, githubToken } from "../github/auth.ts"; import { github } from "../github/index.ts"; import { $ } from "../machine/command.ts"; -import { destroyOrphans, destroyRunMachines } from "../machine/machine.ts"; +import { + destroyOrphans, + destroyRunMachines, + machineSetupScript, +} from "../machine/machine.ts"; import { sandbox } from "../machine/sandbox.ts"; -import { machineSetupScript } from "../machine/start.ts"; import { report } from "../report.ts"; import { createCiTestClient } from "../testing/client.ts"; import { prEvent, prTrigger } from "../testing/events.ts"; From 7f9092bab9508f7bfd363bd616a7afe611b307c2 Mon Sep 17 00:00:00 2001 From: x Date: Sat, 10 Oct 2026 00:45:28 +0000 Subject: [PATCH 04/11] refactor(ci): require inngest 4.23.1 and drop the older-SDK paths The span API and a snapshot's own span are always there, so the stub, the version gate and the dual snapshot-span path are gone. The shared test harness had one user, so it is local to that test again. Co-Authored-By: Claude Opus 5.5 --- .changeset/ci-trace-spans.md | 4 +- packages/ci/package.json | 2 +- packages/ci/src/machine/machine.ts | 14 +- packages/ci/src/machine/spans.test.ts | 108 +++++++++++- packages/ci/src/pipeline/README.md | 2 +- packages/ci/src/pipeline/spans.test.ts | 218 ++----------------------- packages/ci/src/pipeline/spans.ts | 92 +---------- packages/ci/src/testing/README.md | 2 - packages/ci/src/testing/harness.ts | 115 ------------- packages/ci/src/testing/runFunction.ts | 14 +- packages/ci/src/testing/setup.ts | 11 -- packages/ci/src/testing/spanStub.ts | 185 --------------------- packages/ci/vitest.config.ts | 1 - 13 files changed, 140 insertions(+), 628 deletions(-) delete mode 100644 packages/ci/src/testing/harness.ts delete mode 100644 packages/ci/src/testing/setup.ts delete mode 100644 packages/ci/src/testing/spanStub.ts diff --git a/.changeset/ci-trace-spans.md b/.changeset/ci-trace-spans.md index abe6dd96e..ac0d50c6a 100644 --- a/.changeset/ci-trace-spans.md +++ b/.changeset/ci-trace-spans.md @@ -2,4 +2,6 @@ "@inngest/ci": minor --- -Group a run's steps into trace spans (one per job, one per command, one per extra machine), name CI's own steps for what they do, and mark them as CI's work so a trace can tell them from yours. Give a job a `name` to change what its span is called. Every span has a kind (job, sandbox, command, snapshot, github or attempt), and each statement on a background process (`output`, `exited`, `kill`) is a row of its own. Saving a sandbox is one `snapshot` row on an `inngest` that takes the span, and the span is opened by CI on older ones. +Group a run's steps into trace spans (one per job, one per command, one per extra machine), name CI's own steps for what they do, and mark them as CI's work so a trace can tell them from yours. Give a job a `name` to change what its span is called. Every span has a kind (job, sandbox, command, snapshot, github or attempt), and each statement on a background process (`output`, `exited`, `kill`) is a row of its own. Saving a sandbox is one `snapshot` row. + +`@inngest/ci` now needs `inngest` 4.23.1 or later, which takes a snapshot's own trace span and always has the span API; older SDKs are no longer supported. diff --git a/packages/ci/package.json b/packages/ci/package.json index 045e77192..0028cd97e 100644 --- a/packages/ci/package.json +++ b/packages/ci/package.json @@ -64,7 +64,7 @@ "node": ">=20" }, "peerDependencies": { - "inngest": "^4.22.0" + "inngest": "^4.23.1" }, "dependencies": { "@octokit/auth-app": "^8.3.1", diff --git a/packages/ci/src/machine/machine.ts b/packages/ci/src/machine/machine.ts index 8d9751a38..9a36a46f1 100644 --- a/packages/ci/src/machine/machine.ts +++ b/packages/ci/src/machine/machine.ts @@ -23,11 +23,7 @@ import { runInScope, scopeSeparator, } from "../pipeline/scope.ts"; -import { - inSnapshotSpan, - inSpan, - snapshotSpanOption, -} from "../pipeline/spans.ts"; +import { inSpan } from "../pipeline/spans.ts"; import type { MachineConfig } from "../types.ts"; import { boundedName, @@ -501,13 +497,13 @@ export interface TakenSnapshot { /** * A snapshot step: its ID is the step's, its name reads as an action. Saving * a job's sandbox is one statement, so its row is the job's "Save sandbox" - * span, which the SDK is told of where it can take it. + * span, which the SDK takes as the step's own so it opens no second group. */ const snapshotStep = (jobPath: string, id: string) => { return { id, name: traceName.snapshotMachine, - ...snapshotSpanOption(spans.save(jobPath)), + "~span": spans.save(jobPath), }; }; @@ -532,9 +528,7 @@ export const snapshotMachine = async ( const handle = await scope.machine; - return inSnapshotSpan(spans.save(scope.path), () => { - return takeSnapshot(run, scope.path, handle, cache); - }); + return takeSnapshot(run, scope.path, handle, cache); }; const takeSnapshot = async ( diff --git a/packages/ci/src/machine/spans.test.ts b/packages/ci/src/machine/spans.test.ts index b88bbaf4d..91847ca11 100644 --- a/packages/ci/src/machine/spans.test.ts +++ b/packages/ci/src/machine/spans.test.ts @@ -10,17 +10,119 @@ * @module */ -import { step } from "inngest"; +import { version as sdkVersion, step } from "inngest"; import { describe, expect, test } from "vitest"; import { CommandFailedError } from "../errors.ts"; +import { consoleReporter } from "../github/auth.ts"; +import { createCi } from "../pipeline/createCi.ts"; import { ciOrigin } from "../pipeline/names.ts"; +import { createCiTestClient } from "../testing/client.ts"; +import { prEvent, prTrigger } from "../testing/events.ts"; +import type { CommandScript, FakeSandboxApi } from "../testing/fakeSandbox.ts"; import { createFakeSandboxApi } from "../testing/fakeSandbox.ts"; -import type { Ci } from "../testing/harness.ts"; -import { ciTest, drawTrace } from "../testing/harness.ts"; +import { runFunction } from "../testing/runFunction.ts"; import { version } from "../version.ts"; import { $ } from "./command.ts"; import { sandbox } from "./sandbox.ts"; +type Ci = ReturnType; + +type RunResult = Awaited>; + +const ciTest = ( + options: { + /** The fake API to run on. Defaults to a fresh one. */ + api?: FakeSandboxApi; + /** What the commands it runs do. */ + scripts?: CommandScript[]; + } = {}, +) => { + const api = options.api ?? createFakeSandboxApi(); + + api.script(options.scripts ?? []); + + const ci = createCi(createCiTestClient(api), { github: consoleReporter() }); + + return { + api, + ci, + /** Run the pipeline whose handler `define` makes from `ci`. */ + run: (define: (ci: Ci) => () => Promise): Promise => { + const pipeline = ci.pipeline({ id: "pr", on: prTrigger }, define(ci)); + + return runFunction(pipeline, { event: prEvent }); + }, + }; +}; + +interface TraceNode { + label: string; + children: Map; +} + +/** + * Draw a run's trace as an indented tree, the way the UI nests it: each span + * once, where its first step is, with its kind in brackets, and each step + * under its spans. + */ +const drawTrace = ( + result: RunResult, + /** Suffix each row with who it says did it, as in `Create sandbox <- ci`. */ + withOrigins = false, +): string => { + const root: TraceNode = { label: "", children: new Map() }; + + const by = (origin: string | undefined) => { + if (!withOrigins || !origin) { + return ""; + } + + const names: Record = { + [ciOrigin]: "ci", + [`inngest@${sdkVersion}`]: "inngest", + }; + + return ` <- ${names[origin] ?? origin}`; + }; + + for (const stepId of result.stepIds) { + let node = root; + + for (const span of result.spans[stepId] ?? []) { + const key = `span:${span.id}`; + const label = span.kind ? `${span.name} [${span.kind}]` : span.name; + + const child = node.children.get(key) ?? { + label: `${label}${by(span.origin)}`, + children: new Map(), + }; + + node.children.set(key, child); + + node = child; + } + + node.children.set(`step:${stepId}`, { + label: `${result.names[stepId] ?? stepId}${by(result.origins[stepId])}`, + children: new Map(), + }); + } + + const lines: string[] = []; + + const draw = (node: TraceNode, depth: number) => { + for (const child of node.children.values()) { + lines.push(`${" ".repeat(depth)}${child.label}`); + + draw(child, depth + 1); + } + }; + + draw(root, 0); + + return lines.join("\n"); +}; + /** Run a job with a captured, a managed, a retried and a background command. */ const run = () => { return ciTest({ diff --git a/packages/ci/src/pipeline/README.md b/packages/ci/src/pipeline/README.md index d6748126e..7072e7007 100644 --- a/packages/ci/src/pipeline/README.md +++ b/packages/ci/src/pipeline/README.md @@ -11,6 +11,6 @@ The client and the run engine. - `metadata.ts`: the `userland.inngest-ci` metadata attached to runs and steps, and the helpers that build and attach it. - `scope.ts`: the run and job scopes held in async context. Jobs only write to the run scope, except for `builds`, the promises of the builds `from` parents need. - `names.ts`: what each step and span CI writes is called in the trace, and the origin that marks CI's own work. -- `spans.ts`: the one place CI touches the SDK's experimental trace-span API. It is feature-detected, so CI runs unchanged on an SDK without it. +- `spans.ts`: the one place CI touches the SDK's experimental trace-span API. - `durable.ts`: durable proxies that run calls as steps. - `rerun.ts`: re-running a pipeline from a GitHub check. diff --git a/packages/ci/src/pipeline/spans.test.ts b/packages/ci/src/pipeline/spans.test.ts index 100f1de8c..6e5f13b71 100644 --- a/packages/ci/src/pipeline/spans.test.ts +++ b/packages/ci/src/pipeline/spans.test.ts @@ -1,131 +1,16 @@ /** - * Tests of the span helpers, and of CI on an SDK without the span API: with - * none, functions still run, step IDs and names are exactly what they are - * with it, and the trace just has no spans or origins. The same goes for an - * SDK that doesn't take a snapshot's own span. The rest of the suite runs - * with the test stub in place. + * Tests of the span helpers, and of the one span a snapshot is: CI hands it + * to the SDK as the step's own, so the SDK opens no second group around the + * snapshot's create and wait. * * @module */ -import { group, step } from "inngest"; -import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -import { $ } from "../machine/command.ts"; +import { describe, expect, test, vi } from "vitest"; import { snapshotMachine } from "../machine/machine.ts"; -import { ciTest } from "../testing/harness.ts"; -import { installSpanStub, removeSpanStub } from "../testing/spanStub.ts"; import { ciOrigin } from "./names.ts"; import type { CiJobScope } from "./scope.ts"; -import { - hasSpanApi, - inSnapshotSpan, - inSpan, - originOption, - sdk, - snapshotSpanOption, - versionAtLeast, -} from "./spans.ts"; - -/** Run a job with a command and a step of your own. */ -const run = () => { - return ciTest().run((ci) => { - return ci.job("test", async () => { - await $`pnpm test`; - - await step.run("mine", () => { - return "done"; - }); - }); - }); -}; - -const span = { id: "a", name: "A", kind: "job" } as const; - -describe("without the SDK's span API", () => { - beforeEach(() => { - removeSpanStub(); - }); - - afterEach(() => { - installSpanStub(); - }); - - test("is detected as absent", () => { - expect(hasSpanApi()).toBe(false); - - expect("~span" in group).toBe(false); - }); - - test("inSpan runs the function and returns what it returns", async () => { - expect(inSpan(span, () => 7)).toBe(7); - - await expect( - inSpan(span, () => { - return Promise.resolve("later"); - }), - ).resolves.toBe("later"); - - expect(() => { - return inSpan(span, () => { - throw new Error("from fn"); - }); - }).toThrow("from fn"); - }); - - test("steps keep their IDs and names, and the trace has no spans", async () => { - installSpanStub(); - - const withSpans = await run(); - - removeSpanStub(); - - const without = await run(); - - expect(without.type).toBe("function-resolved"); - - expect(without.stepIds).toEqual(withSpans.stepIds); - expect(without.names).toEqual(withSpans.names); - - expect(without.stepIds).toContain("test › pnpm test › start"); - expect(without.names["test › pnpm test › start"]).toBe("Start process"); - - expect(Object.keys(withSpans.spans).length).toBeGreaterThan(0); - expect(without.spans).toEqual({}); - - // Origins aren't asserted absent: an SDK with the span API stamps a - // step's `~origin` option itself, whether or not `group["~span"]` is there - // to open spans, and this suite runs against that SDK. - }); -}); - -describe("with the SDK's span API", () => { - test("is detected, and inSpan hands it the span and the function", () => { - expect(hasSpanApi()).toBe(true); - - removeSpanStub(); - - const calls: unknown[] = []; - - Object.defineProperty(group, "~span", { - value: (opened: unknown, fn: () => unknown) => { - calls.push(opened); - - return `wrapped ${String(fn())}`; - }, - configurable: true, - }); - - try { - expect(inSpan(span, () => "inner")).toBe("wrapped inner"); - - expect(calls).toEqual([span]); - } finally { - Reflect.deleteProperty(group, "~span"); - - installSpanStub(); - } - }); -}); +import { originOption } from "./spans.ts"; describe("originOption", () => { test("is an extra key on the step options, and nothing else", () => { @@ -137,39 +22,9 @@ describe("originOption", () => { }); }); -describe("versionAtLeast", () => { - test.each([ - ["4.23.1", true], - ["4.23.2", true], - ["4.24.0", true], - ["5.0.0", true], - ["4.23.1-pr.1", true], - ["4.23.0", false], - ["4.22.9", false], - ["3.99.99", false], - ])("%s", (version, expected) => { - expect(versionAtLeast(version, "4.23.1")).toBe(expected); - }); -}); - describe("a snapshot's span", () => { - /** What a job's snapshot passes the SDK, and the spans opened around it. */ - const snapshotWith = async (takesSpan: boolean) => { - vi.spyOn(sdk, "takesSnapshotSpan").mockReturnValue(takesSpan); - + test("is the job's Save sandbox span, passed as the step's own", async () => { const steps: unknown[] = []; - const opened: unknown[] = []; - - const open = vi.fn((spanned: unknown, fn: () => unknown) => { - opened.push(spanned); - - return fn(); - }); - - Object.defineProperty(group, "~span", { - value: open, - configurable: true, - }); const scope = { path: "test", @@ -185,59 +40,18 @@ describe("a snapshot's span", () => { run: { ci: { hooks: { activity: vi.fn() } } }, } as unknown as CiJobScope; - try { - await snapshotMachine(scope); - } finally { - Reflect.deleteProperty(group, "~span"); - - installSpanStub(); - - vi.restoreAllMocks(); - } - - return { steps, opened }; - }; - - const save = { - id: "test › save", - name: "Save sandbox", - kind: "snapshot", - origin: ciOrigin, - }; - - test("goes to an SDK that takes it, as the step's own span", async () => { - removeSpanStub(); - - const { steps, opened } = await snapshotWith(true); + await snapshotMachine(scope); expect(steps).toEqual([ - expect.objectContaining({ id: "test › snapshot", "~span": save }), + expect.objectContaining({ + id: "test › snapshot", + "~span": { + id: "test › save", + name: "Save sandbox", + kind: "snapshot", + origin: ciOrigin, + }, + }), ]); - - expect(opened).toEqual([]); - }); - - test("is opened here for an SDK that doesn't, which would open its own", async () => { - removeSpanStub(); - - const { steps, opened } = await snapshotWith(false); - - expect(steps).toEqual([expect.not.objectContaining({ "~span": save })]); - - expect(opened).toEqual([save]); - }); - - test("is a step option only where the SDK takes it", () => { - vi.spyOn(sdk, "takesSnapshotSpan").mockReturnValue(true); - - expect(snapshotSpanOption(span)).toEqual({ "~span": span }); - - vi.spyOn(sdk, "takesSnapshotSpan").mockReturnValue(false); - - expect(snapshotSpanOption(span)).toEqual({}); - - expect(inSnapshotSpan(span, () => "ran")).toBe("ran"); - - vi.restoreAllMocks(); }); }); diff --git a/packages/ci/src/pipeline/spans.ts b/packages/ci/src/pipeline/spans.ts index b2471c28e..57ad8e745 100644 --- a/packages/ci/src/pipeline/spans.ts +++ b/packages/ci/src/pipeline/spans.ts @@ -1,16 +1,13 @@ /** - * The one place CI touches the SDK's experimental trace-span API. The span - * API ships in newer SDKs than the oldest one CI supports, so it's - * feature-detected here: where the SDK has it, spans and origins mark up the - * trace, and where it doesn't, `inSpan` just runs the function and the origin - * step option is an unknown key the SDK ignores. Step IDs and names never - * depend on it. Everything else in CI goes through these helpers rather than - * reaching for the SDK's span API itself. + * The one place CI touches the SDK's experimental trace-span API: spans group + * the steps run inside them in the trace, and an origin marks work as CI's. + * Step IDs and names never depend on either. Everything else in CI goes + * through these helpers rather than reaching for the SDK's span API itself. * * @module */ -import { group, version as sdkVersion } from "inngest"; +import { group } from "inngest"; /** What a span stands for. Every span CI opens has one. */ export type SpanKind = @@ -33,93 +30,20 @@ export interface SpanInfo { origin?: string; } -/** The part of the SDK's `group` this file uses, when the SDK has it. */ -interface SpanApi { - "~span"?: (span: SpanInfo, fn: () => R) => R; -} - -const spanApi = (): SpanApi => { - return group as unknown as SpanApi; -}; - -/** Whether the SDK in use has the span API. */ -export const hasSpanApi = (): boolean => { - return typeof spanApi()["~span"] === "function"; -}; - -/** - * Run `fn` in a trace span, or just run it when the SDK has no span API. - */ +/** Run `fn` in a trace span. */ export const inSpan = ( /** The span to run `fn` in. */ span: SpanInfo, /** What to run. */ fn: () => R, ): R => { - const api = spanApi(); - const open = api["~span"]; - - if (typeof open !== "function") { - return fn(); - } - - return open.call(api, span, fn) as R; + return group["~span"](span, fn); }; -/** - * The step option that marks a step as work `origin` did. A step option is - * just a key on the options object, and an SDK without the span API ignores - * it, so it's safe to always pass. - */ +/** The step option that marks a step as work `origin` did. */ export const originOption = ( /** Who did the work. */ origin: string, ): { "~origin": string } => { return { "~origin": origin }; }; - -/** The first SDK whose snapshot takes a caller's `"~span"` as its own span. */ -const snapshotSpanSince = "4.23.1"; - -/** Whether `version` is `since` or later, as in `4.23.1`. */ -export const versionAtLeast = (version: string, since: string): boolean => { - const have = version.split(/[.-]/).map(Number); - - for (const [index, want] of since.split(".").map(Number).entries()) { - const got = have[index] ?? 0; - - if (got !== want) { - return got > want; - } - } - - return true; -}; - -/** - * What the SDK in use can do beyond the span API. Older SDKs ignore the step - * options they don't know, so each is detected here rather than assumed. - */ -export const sdk = { - /** - * Whether a snapshot step's `"~span"` option stands in for the span the SDK - * would open around its create and wait. Older SDKs ignore the option and - * open their own, which then sits inside the span CI opens. - */ - takesSnapshotSpan: (): boolean => { - return versionAtLeast(sdkVersion, snapshotSpanSince); - }, -}; - -/** - * The step option that makes a snapshot one statement in `span`, where the - * SDK takes one. Elsewhere there is none, and `inSnapshotSpan` opens it. - */ -export const snapshotSpanOption = (span: SpanInfo): { "~span"?: SpanInfo } => { - return sdk.takesSnapshotSpan() ? { "~span": span } : {}; -}; - -/** Run `fn`, which takes snapshots, in `span` unless the SDK does it itself. */ -export const inSnapshotSpan = (span: SpanInfo, fn: () => R): R => { - return sdk.takesSnapshotSpan() ? fn() : inSpan(span, fn); -}; diff --git a/packages/ci/src/testing/README.md b/packages/ci/src/testing/README.md index bcbca1697..688f4b139 100644 --- a/packages/ci/src/testing/README.md +++ b/packages/ci/src/testing/README.md @@ -6,7 +6,5 @@ Test-only helpers. Not exported and not built. - `fakeGitHub.ts`: a fake GitHub HTTP layer. - `client.ts`: an Inngest client wired to the fake sandbox API, which also records the functions it creates. - `events.ts`: the pull request event and trigger tests share. -- `spanStub.ts`, `setup.ts`: a stand-in for the SDK's span API, installed before every test file when the SDK has none, so span and origin assertions hold on any SDK. -- `harness.ts`: `ciTest`, a client on the fake API with a `run` for a pipeline a test defines, and `drawTrace` for the trace it left. - `runFunction.ts`: drives a function to completion like the executor, including `step.invoke` of the functions the test client created. Invokes run beside the function, and the function is called again as each one ends; runs of one `event.data` concurrency key take turns. - `schema.ts`: a fake Standard Schema that can also write itself as JSON Schema. diff --git a/packages/ci/src/testing/harness.ts b/packages/ci/src/testing/harness.ts deleted file mode 100644 index 4fa5142c2..000000000 --- a/packages/ci/src/testing/harness.ts +++ /dev/null @@ -1,115 +0,0 @@ -/** - * What a test of a pipeline starts from: a client on the fake Sandboxes API - * and a `ci` that reports to the console, with `run` driving one pipeline - * whose handler a test defines. `drawTrace` draws the trace such a run left. - * - * @module - */ - -import { version as sdkVersion } from "inngest"; -import { consoleReporter } from "../github/auth.ts"; -import { createCi } from "../pipeline/createCi.ts"; -import { ciOrigin } from "../pipeline/names.ts"; -import { createCiTestClient } from "./client.ts"; -import { prEvent, prTrigger } from "./events.ts"; -import type { CommandScript, FakeSandboxApi } from "./fakeSandbox.ts"; -import { createFakeSandboxApi } from "./fakeSandbox.ts"; -import { runFunction } from "./runFunction.ts"; - -export type Ci = ReturnType; - -export type RunResult = Awaited>; - -export const ciTest = ( - options: { - /** The fake API to run on. Defaults to a fresh one. */ - api?: FakeSandboxApi; - /** What the commands it runs do. */ - scripts?: CommandScript[]; - } = {}, -) => { - const api = options.api ?? createFakeSandboxApi(); - - api.script(options.scripts ?? []); - - const ci = createCi(createCiTestClient(api), { github: consoleReporter() }); - - return { - api, - ci, - /** Run the pipeline whose handler `define` makes from `ci`. */ - run: (define: (ci: Ci) => () => Promise): Promise => { - const pipeline = ci.pipeline({ id: "pr", on: prTrigger }, define(ci)); - - return runFunction(pipeline, { event: prEvent }); - }, - }; -}; - -interface TraceNode { - label: string; - children: Map; -} - -/** - * Draw a run's trace as an indented tree, the way the UI nests it: each span - * once, where its first step is, with its kind in brackets, and each step - * under its spans. - */ -export const drawTrace = ( - result: RunResult, - /** Suffix each row with who it says did it, as in `Create sandbox <- ci`. */ - withOrigins = false, -): string => { - const root: TraceNode = { label: "", children: new Map() }; - - const by = (origin: string | undefined) => { - if (!withOrigins || !origin) { - return ""; - } - - const names: Record = { - [ciOrigin]: "ci", - [`inngest@${sdkVersion}`]: "inngest", - }; - - return ` <- ${names[origin] ?? origin}`; - }; - - for (const stepId of result.stepIds) { - let node = root; - - for (const span of result.spans[stepId] ?? []) { - const key = `span:${span.id}`; - const label = span.kind ? `${span.name} [${span.kind}]` : span.name; - - const child = node.children.get(key) ?? { - label: `${label}${by(span.origin)}`, - children: new Map(), - }; - - node.children.set(key, child); - - node = child; - } - - node.children.set(`step:${stepId}`, { - label: `${result.names[stepId] ?? stepId}${by(result.origins[stepId])}`, - children: new Map(), - }); - } - - const lines: string[] = []; - - const draw = (node: TraceNode, depth: number) => { - for (const child of node.children.values()) { - lines.push(`${" ".repeat(depth)}${child.label}`); - - draw(child, depth + 1); - } - }; - - draw(root, 0); - - return lines.join("\n"); -}; diff --git a/packages/ci/src/testing/runFunction.ts b/packages/ci/src/testing/runFunction.ts index 52b6f1085..29c461508 100644 --- a/packages/ci/src/testing/runFunction.ts +++ b/packages/ci/src/testing/runFunction.ts @@ -15,7 +15,6 @@ import type { EventPayload, InngestFunction } from "inngest"; import { createdFunctions } from "./client.ts"; -import { SpanStubMiddleware, spanStubInstalled, stampOf } from "./spanStub.ts"; /** The opcodes the harness reads; their values are the wire format. */ const StepOpCode = { @@ -340,12 +339,8 @@ export const runFunction = async ( steps[id] = step.data; - // Where the SDK has the span API it stamps these on the step. Where the - // test stub provides it, the stub recorded them instead. - const stamp = spanStubInstalled() ? stampOf(id) : undefined; - const span = - step.opts?.span ?? (stamp?.span.length ? stamp.span : undefined); - const origin = step.opts?.origin ?? stamp?.origin; + // The SDK stamps a step's spans and origin on it. + const { span, origin } = step.opts ?? {}; if (span) { spans[id] = span; @@ -580,11 +575,6 @@ const runOnce = async ( return new Middleware({ client }); }); - if (spanStubInstalled()) { - // First, so the other middleware builds on step tools that record. - middlewareInstances.unshift(new SpanStubMiddleware({ client })); - } - const execution = internals["createExecution"]({ partialOptions: { client, diff --git a/packages/ci/src/testing/setup.ts b/packages/ci/src/testing/setup.ts deleted file mode 100644 index f7532a0aa..000000000 --- a/packages/ci/src/testing/setup.ts +++ /dev/null @@ -1,11 +0,0 @@ -/** - * Runs before every test file: gives the SDK the span API when it has none, - * so span and origin assertions hold against any SDK. A test of the no-span - * path takes it off again with `removeSpanStub`. - * - * @module - */ - -import { installSpanStub } from "./spanStub.ts"; - -installSpanStub(); diff --git a/packages/ci/src/testing/spanStub.ts b/packages/ci/src/testing/spanStub.ts deleted file mode 100644 index 481979fbe..000000000 --- a/packages/ci/src/testing/spanStub.ts +++ /dev/null @@ -1,185 +0,0 @@ -/** - * A stand-in for the SDK's experimental trace-span API, for tests that run - * against an SDK that doesn't have it yet. It does what the real one does - * where tests can see it: a span opened with `inSpan` is kept in the async - * context, so a step called inside it is recorded under the span path, and a - * step's origin is its own `"~origin"` option or the innermost span's. Where - * the real SDK stamps a step's `opts.span` and `opts.origin` onto the step - * the executor reports, the stub records them by step ID for `runFunction` to - * read back. It installs nothing when the SDK has the span API itself. - * - * @module - */ - -import { group, Middleware } from "inngest"; -import type { AsyncContext } from "inngest/experimental"; -import { runWithAsyncCtx } from "inngest/experimental"; -import type { SpanInfo } from "../pipeline/spans.ts"; -import { hasSpanApi } from "../pipeline/spans.ts"; - -/** What the stub knows about a step: its spans, outermost first, and origin. */ -export interface StepStamp { - span: SpanInfo[]; - origin?: string; -} - -/** Where a span stack lives in an async context. Contexts copied by spread keep it. */ -const spanStackKey = Symbol("ci.spanStub.spans"); - -interface SpannedContext extends AsyncContext { - [spanStackKey]?: SpanInfo[]; -} - -const stamps = new Map(); - -let installed = false; - -/** - * The current async context, read synchronously like the real span API does. - * The SDK keeps its async storage on a global, and only offers an async getter, - * which would delay every span by a tick and reorder concurrent work. - */ -const currentContext = (): SpannedContext | undefined => { - const cache = ( - globalThis as Record< - symbol, - { resolved?: { getStore: () => AsyncContext | undefined } } | undefined - > - )[Symbol.for("inngest:als")]; - - return cache?.resolved?.getStore(); -}; - -const spanOf = (span: SpanInfo, fn: () => R): R => { - const ctx = currentContext(); - - if (!ctx) { - return fn(); - } - - const spans = [...(ctx[spanStackKey] ?? []), span]; - - const spanned: SpannedContext = { ...ctx, [spanStackKey]: spans }; - - return runWithAsyncCtx(spanned, fn); -}; - -/** The SDK's own `"~span"`, while `removeSpanStub` has it taken off. */ -let hidden: PropertyDescriptor | undefined; - -/** Give the SDK's `group` a `"~span"`, unless it already has one. */ -export const installSpanStub = (): void => { - if (hidden) { - Object.defineProperty(group, "~span", hidden); - - hidden = undefined; - - return; - } - - if (hasSpanApi()) { - return; - } - - Object.defineProperty(group, "~span", { - value: spanOf, - configurable: true, - writable: true, - }); - - installed = true; -}; - -/** - * Take the stub off again, so the SDK looks as it does without the span API. - * An SDK that has the span API itself has it taken off too, until - * `installSpanStub` puts it back. - */ -export const removeSpanStub = (): void => { - if (installed) { - Reflect.deleteProperty(group, "~span"); - - installed = false; - - return; - } - - hidden ??= Object.getOwnPropertyDescriptor(group, "~span"); - - Reflect.deleteProperty(group, "~span"); -}; - -/** Whether the stub is what's providing the span API. */ -export const spanStubInstalled = (): boolean => { - return installed; -}; - -/** What the stub recorded for a step, by the ID CI gave it. */ -export const stampOf = (stepId: string): StepStamp | undefined => { - return stamps.get(stepId); -}; - -/** Record the step `args` is about to run, from the span stack in effect now. */ -const record = (args: unknown[]): void => { - const first = args[0]; - - const options = - typeof first === "string" - ? { id: first } - : (first as { id?: unknown; "~origin"?: unknown } | undefined); - - if (typeof options?.id !== "string") { - return; - } - - const span = currentContext()?.[spanStackKey] ?? []; - - const inherited = [...span].reverse().find((entry) => { - return entry.origin; - })?.origin; - - const own = (options as { "~origin"?: unknown })["~origin"]; - - stamps.set(options.id, { - span, - origin: typeof own === "string" ? own : inherited, - }); -}; - -/** - * Wrap step tools so every call is recorded. The SDK's middleware hooks don't - * hand a step's unknown options to middleware, so the calls are watched where - * they're made. - */ -const recording = (tools: T): T => { - return new Proxy(tools, { - get(target, prop, receiver) { - const value = Reflect.get(target, prop, receiver) as unknown; - - if (typeof value === "function") { - return (...args: unknown[]) => { - record(args); - - return (value as (...a: unknown[]) => unknown).apply(target, args); - }; - } - - if (value && typeof value === "object") { - return recording(value); - } - - return value; - }, - }); -}; - -/** Records each step's span path and origin as it's called. */ -export class SpanStubMiddleware extends Middleware.BaseMiddleware { - readonly id = "ci-span-stub"; - - override transformFunctionInput( - arg: Middleware.TransformFunctionInputArgs, - ): Middleware.TransformFunctionInputArgs { - return { ...arg, ctx: { ...arg.ctx, step: recording(arg.ctx.step) } }; - } -} diff --git a/packages/ci/vitest.config.ts b/packages/ci/vitest.config.ts index d3b49e53c..2a6163f50 100644 --- a/packages/ci/vitest.config.ts +++ b/packages/ci/vitest.config.ts @@ -4,7 +4,6 @@ export default defineConfig({ test: { environment: "node", silent: "passed-only", - setupFiles: ["src/testing/setup.ts"], typecheck: { enabled: true, include: ["src/**/*.test.ts"], From 98f6ad1d50cc6d8b30fe98d57638909708e8bce6 Mon Sep 17 00:00:00 2001 From: x Date: Tue, 13 Oct 2026 17:10:43 +0000 Subject: [PATCH 05/11] refactor(ci): share step names, sandbox admin, snapshot names and cache lookups CI's step IDs and names move to one catalog in pipeline/names.ts, run through a thin ciRun helper that tags the step as before. Sandbox paging and snapshot deletes (machine/admin.ts), snapshot names (cache/names.ts), the cache lookup and the check-complete payload each have one implementation. No step ID, name or wire format changes. Co-Authored-By: Claude Opus 5.5 --- .changeset/ci-shared-helpers.md | 5 + packages/ci/src/cache/cache.ts | 132 +++++---- packages/ci/src/cache/names.ts | 25 ++ packages/ci/src/checkout/changed.ts | 12 +- packages/ci/src/checkout/checkout.ts | 11 +- packages/ci/src/github/checks.ts | 218 ++++++--------- packages/ci/src/github/helpers.ts | 7 +- packages/ci/src/machine/admin.test.ts | 70 +++++ packages/ci/src/machine/admin.ts | 75 +++++ packages/ci/src/machine/command.ts | 12 +- packages/ci/src/machine/machine.ts | 130 +++------ packages/ci/src/pipeline/durable.ts | 6 +- packages/ci/src/pipeline/job.ts | 31 +-- packages/ci/src/pipeline/metadata.ts | 34 ++- packages/ci/src/pipeline/names.ts | 301 +++++++++++++++++---- packages/ci/src/pipeline/pipeline.ts | 51 ++-- packages/ci/src/pipeline/stepNames.test.ts | 66 +++++ packages/ci/src/report.ts | 7 +- packages/ci/src/units.test.ts | 33 ++- 19 files changed, 792 insertions(+), 434 deletions(-) create mode 100644 .changeset/ci-shared-helpers.md create mode 100644 packages/ci/src/cache/names.ts create mode 100644 packages/ci/src/machine/admin.test.ts create mode 100644 packages/ci/src/machine/admin.ts create mode 100644 packages/ci/src/pipeline/stepNames.test.ts diff --git a/.changeset/ci-shared-helpers.md b/.changeset/ci-shared-helpers.md new file mode 100644 index 000000000..400014067 --- /dev/null +++ b/.changeset/ci-shared-helpers.md @@ -0,0 +1,5 @@ +--- +"@inngest/ci": patch +--- + +Internal refactor: CI's step IDs and names come from one catalog, and snapshot naming, sandbox paging and snapshot deletes, cache lookups and check updates share helpers. Nothing user-visible changes. diff --git a/packages/ci/src/cache/cache.ts b/packages/ci/src/cache/cache.ts index 173b327c5..59f072bdd 100644 --- a/packages/ci/src/cache/cache.ts +++ b/packages/ci/src/cache/cache.ts @@ -10,8 +10,9 @@ * @module */ -import { tagStep } from "../pipeline/metadata.ts"; -import { ciStep, traceName } from "../pipeline/names.ts"; +import { ciRun } from "../pipeline/metadata.ts"; +import type { StepSpec } from "../pipeline/names.ts"; +import { steps } from "../pipeline/names.ts"; import type { CiJobScope, CiRunScope } from "../pipeline/scope.ts"; import { countApi, joinId, rootRunIdOf } from "../pipeline/scope.ts"; import type { @@ -22,12 +23,12 @@ import type { RepoContext, } from "../types.ts"; import { - boundedName, formatRelative, hash, isSnapshotNotFound, stableStringify, } from "../util.ts"; +import { formatName } from "./names.ts"; /** * Where a job reads and writes its cached snapshots. @@ -82,19 +83,6 @@ const sharedScopes = ( return { read: [branch], write: branch }; }; -/** - * The name a job's cached snapshot has in a scope: `ci///`. - * A name too long for a snapshot keeps its start and gains a hash of the - * whole, so it stays unique. - */ -export const snapshotName = ( - scope: string, - jobId: string, - ownKey: string, -): string => { - return boundedName(`ci/${scope}/${jobId}/${ownKey}`); -}; - /** * Resolve the parts of a cache key into a single hash. * @@ -282,18 +270,18 @@ export const cacheTarget = async ( countApi("cache"); - const ownKey = (await run.step.run( - ciStep(joinId(job.path, "cache:key"), traceName.checkCache), - async () => { - await tagStep(run, { kind: "cache", job: job.path }); - - return jobCacheKey(run, cache, input, base); - }, - )) as string; + const ownKey = await ciRun(run, steps.cacheKey(job.path), () => { + return jobCacheKey(run, cache, input, base); + }); return { ownKey, - name: snapshotName(cacheScopes(run.repo, cache.scope).write, jobId, ownKey), + name: formatName({ + kind: "cache", + scope: cacheScopes(run.repo, cache.scope).write, + jobId, + ownKey, + }), }; }; @@ -314,7 +302,12 @@ export const runTarget = ( return { ownKey, - name: snapshotName(`run:${rootRunIdOf(run)}`, jobId, ownKey), + name: formatName({ + kind: "run", + rootRunId: rootRunIdOf(run), + jobId, + ownKey, + }), }; }; @@ -435,7 +428,7 @@ const findInScopes = async ( for (const scope of cacheScopes(run.repo, cache.scope).read) { const found = await findNamed( run, - snapshotName(scope, jobId, ownKey), + formatName({ kind: "cache", scope, jobId, ownKey }), exclude, ); @@ -459,24 +452,28 @@ export const lookupCache = async ( /** A snapshot found to be bad, which a rebuild must not find again. */ exclude?: string, ): Promise => { - const { run } = scope; - - const found = (await run.step.run( - ciStep(joinId(scope.path, "cache:lookup"), traceName.lookUpCache), - async () => { - await tagStep(run, { kind: "cache", job: scope.path }); - - const hit = await findCached( - run, - scope.config.id, - cache, - target, - exclude, - ); - - return hit ?? null; - }, - )) as CachedSnapshot | null; + return lookUp( + scope.run, + steps.lookUpCache(joinId(scope.path, "cache:lookup"), scope.path), + scope.config.id, + cache, + target, + exclude, + ); +}; + +/** A lookup as a memoized step: a hit is returned, a miss is `null` in the step. */ +const lookUp = async ( + run: CiRunScope, + spec: StepSpec, + jobId: string, + cache: CacheConfig | undefined, + target: CacheTarget, + exclude?: string, +): Promise => { + const found = await ciRun(run, spec, async () => { + return (await findCached(run, jobId, cache, target, exclude)) ?? null; + }); return found ?? undefined; }; @@ -515,18 +512,14 @@ export const lookupBeforeBuild = async ( /** A snapshot found to be bad, which a rebuild must not find again. */ exclude?: string, ): Promise => { - const found = (await run.step.run( - ciStep(joinId(job.stepPath, "lookup"), traceName.lookUpCache), - async () => { - await tagStep(run, { kind: "cache", job: job.path }); - - const hit = await findCached(run, job.id, cache, target, exclude); - - return hit ?? null; - }, - )) as CachedSnapshot | null; - - return found ?? undefined; + return lookUp( + run, + steps.lookUpCache(joinId(job.stepPath, "lookup"), job.path), + job.id, + cache, + target, + exclude, + ); }; /** @@ -546,9 +539,10 @@ export const resolveTakenName = async ( */ broken?: boolean, ): Promise<{ winner?: CachedSnapshot; cleared: boolean }> => { - return (await run.step.run( - ciStep(stepId, traceName.resolveCacheName), - async () => { + return ciRun( + run, + steps.resolveCacheName(stepId), + async (): Promise<{ winner?: CachedSnapshot; cleared: boolean }> => { const winner = await findNamed(run, name, exclude); if (winner) { @@ -582,7 +576,7 @@ export const resolveTakenName = async ( return { cleared }; }, - )) as { winner?: CachedSnapshot; cleared: boolean }; + ); }; /** @@ -596,9 +590,10 @@ export const snapshotState = async ( stepId: string, snapshotId: string, ): Promise<"gone" | "creating" | "ready"> => { - return (await run.step.run( - ciStep(stepId, traceName.checkSnapshotState), - async () => { + return ciRun( + run, + steps.checkSnapshotState(stepId), + async (): Promise<"gone" | "creating" | "ready"> => { try { const snapshot = (await snapshotsClient(run).get(snapshotId)) as | SnapshotResource @@ -614,7 +609,7 @@ export const snapshotState = async ( return "ready"; } }, - )) as "gone" | "creating" | "ready"; + ); }; /** @@ -628,8 +623,9 @@ export const deleteSnapshot = async ( snapshotId: string, ): Promise => { try { - const result = (await run.step.run( - ciStep(stepId, traceName.deleteBadSnapshot), + const result = await ciRun( + run, + steps.deleteBadSnapshot(stepId), async () => { try { const snapshot = await snapshotsClient(run).get(snapshotId); @@ -642,7 +638,7 @@ export const deleteSnapshot = async ( return { deleted: false, gone: isSnapshotNotFound(error) }; } }, - )) as { deleted: boolean; gone: boolean }; + ); // Outside the step, which a replay doesn't run: the set is rebuilt on // every replay, so a snapshot gone now must leave it on each of them. diff --git a/packages/ci/src/cache/names.ts b/packages/ci/src/cache/names.ts new file mode 100644 index 000000000..b3552f1b6 --- /dev/null +++ b/packages/ci/src/cache/names.ts @@ -0,0 +1,25 @@ +/** + * The names snapshots are found by. Everything that builds a snapshot's name + * goes through here, so a change to the naming (such as named images) is a + * change to this file. + * + * @module + */ + +import { boundedName } from "../util.ts"; + +/** What a snapshot's name says it is. */ +export type SnapshotName = + | { kind: "cache"; scope: string; jobId: string; ownKey: string } + | { kind: "run"; rootRunId: string; jobId: string; ownKey: string }; + +/** + * The name a snapshot has: `ci///`, where a run's own + * snapshots have the scope `run:`. A name too long for a snapshot + * keeps its start and gains a hash of the whole, so it stays unique. + */ +export const formatName = (name: SnapshotName): string => { + const scope = name.kind === "cache" ? name.scope : `run:${name.rootRunId}`; + + return boundedName(`ci/${scope}/${name.jobId}/${name.ownKey}`); +}; diff --git a/packages/ci/src/checkout/changed.ts b/packages/ci/src/checkout/changed.ts index 8fad40dce..b88829833 100644 --- a/packages/ci/src/checkout/changed.ts +++ b/packages/ci/src/checkout/changed.ts @@ -6,7 +6,8 @@ */ import { CiUsageError } from "../errors.ts"; -import { traceName } from "../pipeline/names.ts"; +import { ciRun } from "../pipeline/metadata.ts"; +import { steps } from "../pipeline/names.ts"; import type { CiRunScope } from "../pipeline/scope.ts"; import { countApi, getRunScope } from "../pipeline/scope.ts"; import type { RepoContext } from "../types.ts"; @@ -91,9 +92,10 @@ export const changedFiles = async (): Promise => { }; const readChangedFiles = async (run: CiRunScope): Promise => { - const files = (await run.step.run( - { id: "changed", name: traceName.findChangedFiles }, - async () => { + const files = await ciRun( + run, + steps.findChangedFiles(), + async (): Promise => { try { return await listChangedFiles(run.repo); } catch (error) { @@ -106,7 +108,7 @@ const readChangedFiles = async (run: CiRunScope): Promise => { return { unknown: true as const, reason: error.message }; } }, - )) as string[] | { unknown: true; reason: string }; + ); if (!Array.isArray(files)) { run.warnings.push( diff --git a/packages/ci/src/checkout/checkout.ts b/packages/ci/src/checkout/checkout.ts index cb8256309..e0c7198c1 100644 --- a/packages/ci/src/checkout/checkout.ts +++ b/packages/ci/src/checkout/checkout.ts @@ -7,7 +7,8 @@ import { CiUsageError } from "../errors.ts"; import { ensureMachine } from "../machine/machine.ts"; -import { traceName } from "../pipeline/names.ts"; +import { ciRun } from "../pipeline/metadata.ts"; +import { steps } from "../pipeline/names.ts"; import type { CiRunScope, MachineHandle } from "../pipeline/scope.ts"; import { countApi, @@ -91,11 +92,9 @@ export const checkout = async (opts: CheckoutOptions = {}): Promise => { const machine = await ensureMachine(scope); const stepId = joinId(scope.path, "checkout"); - await run.step.run( - { - id: stepId, - name: local ? traceName.uploadWorkingTree : traceName.cloneRepository, - }, + await ciRun( + run, + local ? steps.uploadWorkingTree(stepId) : steps.cloneRepository(stepId), async () => { if (local) { return uploadWorkingTree(run, machine, local.path, target); diff --git a/packages/ci/src/github/checks.ts b/packages/ci/src/github/checks.ts index f6a36c54e..e03d21515 100644 --- a/packages/ci/src/github/checks.ts +++ b/packages/ci/src/github/checks.ts @@ -5,8 +5,8 @@ * @module */ -import { type StepTag, tagStep } from "../pipeline/metadata.ts"; -import { traceName } from "../pipeline/names.ts"; +import { ciRun } from "../pipeline/metadata.ts"; +import { type StepSpec, steps } from "../pipeline/names.ts"; import type { CiRunScope } from "../pipeline/scope.ts"; import { inGitHubSpan } from "../pipeline/scope.ts"; import type { CheckAnnotation, CheckConclusion } from "../types.ts"; @@ -230,11 +230,12 @@ export const normaliseAnnotation = ( /** Run a check update as a step in the run's GitHub span. */ const githubStep = ( run: CiRunScope, - step: { id: string; name: string }, + spec: StepSpec, fn: () => Promise, + runValues?: () => Record, ): Promise => { return inGitHubSpan(run, () => { - return run.step.run(step, fn) as Promise; + return ciRun(run, spec, fn, runValues); }); }; @@ -271,21 +272,44 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return checkRunId === undefined ? {} : { checkRunId }; }; - const start = async ( + /** What `sink.complete` is given for a finished check. */ + const completeArgs = ( run: CiRunScope, key: string, name: string, - step: { id: string; name: string }, - tag: StepTag, + read: CheckResult, + ) => { + return { + run, + name, + ...identity(run, key), + conclusion: read.conclusion, + title: read.title, + summary: truncateSummary(read.summary ?? ""), + annotations: (read.annotations ?? []).map(normaliseAnnotation), + ...idFor(run, key), + }; + }; + + /** A job's check, or the pipeline's when there's no `job`. */ + const start = async ( + run: CiRunScope, + name: string, + job?: string, metadata?: () => Record, ) => { - const result = await githubStep(run, step, async () => { - await tagStep(run, tag, metadata?.()); + const key = job ?? "pipeline"; - const started = await sink.start({ run, name, ...identity(run, key) }); + const result = await githubStep( + run, + steps.startCheck(name, job), + async () => { + const started = await sink.start({ run, name, ...identity(run, key) }); - return { ...started, startedAt: Date.now() }; - }); + return { ...started, startedAt: Date.now() }; + }, + metadata, + ); if (result?.id) { checkRunIds.set(idKey(run, key), result.id); @@ -296,31 +320,23 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { const complete = async ( run: CiRunScope, - key: string, name: string, - step: { id: string; name: string }, result: () => CheckResult, - tag: StepTag, + job?: { path: string; conclusion: CheckConclusion }, metadata?: () => Record, ): Promise => { - const endedAt = await githubStep(run, step, async () => { - await tagStep(run, tag, metadata?.()); - - const read = result(); + const key = job?.path ?? "pipeline"; - await sink.complete({ - run, - name, - ...identity(run, key), - conclusion: read.conclusion, - title: read.title, - summary: truncateSummary(read.summary ?? ""), - annotations: (read.annotations ?? []).map(normaliseAnnotation), - ...idFor(run, key), - }); + const endedAt = await githubStep( + run, + steps.completeCheck(name, job), + async () => { + await sink.complete(completeArgs(run, key, name, result())); - return Date.now(); - }); + return Date.now(); + }, + metadata, + ); checkRunIds.delete(idKey(run, key)); @@ -333,17 +349,7 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return undefined; } - return start( - run, - "pipeline", - run.checkName, - { - id: `github › check:${run.checkName}:start`, - name: traceName.createCheck(run.checkName), - }, - { kind: "check" }, - metadata, - ); + return start(run, run.checkName, undefined, metadata); }, pipelineComplete: async ({ run, metadata, result }) => { @@ -351,18 +357,7 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return; } - await complete( - run, - "pipeline", - run.checkName, - { - id: `github › check:${run.checkName}:complete`, - name: traceName.completeCheck(run.checkName), - }, - result, - { kind: "check" }, - metadata, - ); + await complete(run, run.checkName, result, undefined, metadata); }, jobStart: async ({ run, jobPath, name }) => { @@ -370,16 +365,7 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return undefined; } - return start( - run, - jobPath, - jobCheckName(run, jobPath, name), - { - id: `github › check:${jobPath}:start`, - name: traceName.report(jobPath, "started"), - }, - { kind: "check", job: jobPath }, - ); + return start(run, jobCheckName(run, jobPath, name), jobPath); }, jobComplete: async ({ run, jobPath, name, ...result }) => { @@ -389,16 +375,11 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return complete( run, - jobPath, jobCheckName(run, jobPath, name), - { - id: `github › check:${jobPath}:complete`, - name: traceName.report(jobPath, traceName.outcome(result.conclusion)), - }, () => { return result; }, - { kind: "check", job: jobPath }, + { path: jobPath, conclusion: result.conclusion }, ); }, @@ -413,31 +394,21 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return closing(read()); } - const step = { - id: "github › check:jobs:complete", - name: traceName.report("jobs", "ended with the run"), - }; - - const closed = await githubStep(run, step, async () => { - await tagStep(run, { kind: "check" }); + const closed = await githubStep( + run, + steps.completeJobChecks(), + async () => { + const jobs = read(); - const jobs = read(); + for (const { jobPath, name, ...result } of jobs) { + const check = jobCheckName(run, jobPath, name); - for (const { jobPath, name, ...result } of jobs) { - await sink.complete({ - run, - name: jobCheckName(run, jobPath, name), - ...identity(run, jobPath), - conclusion: result.conclusion, - title: result.title, - summary: truncateSummary(result.summary ?? ""), - annotations: (result.annotations ?? []).map(normaliseAnnotation), - ...idFor(run, jobPath), - }); - } + await sink.complete(completeArgs(run, jobPath, check, result)); + } - return closing(jobs); - }); + return closing(jobs); + }, + ); // Outside the step, so a replay also forgets the IDs its memoized start // steps put back. @@ -458,22 +429,14 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { jobPath === undefined ? run.checkName : jobCheckName(run, jobPath, name); - const step = { - id: `github › check:${key}:retry:${run.attempt}`, - name: traceName.report( - jobPath ?? run.checkName, - traceName.retrying(run.attempt + 2, run.maxAttempts), - ), - }; + const step = steps.retryCheck( + checkName, + run.attempt, + run.maxAttempts, + jobPath, + ); await githubStep(run, step, async () => { - await tagStep( - run, - jobPath === undefined - ? { kind: "check" } - : { kind: "check", job: jobPath }, - ); - await sink.update?.({ run, name: checkName, @@ -490,17 +453,9 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return; } - const step = { - id: `github › check:jobs:retry:${run.attempt}`, - name: traceName.report( - "jobs", - traceName.retrying(run.attempt + 2, run.maxAttempts), - ), - }; + const step = steps.retryJobChecks(run.attempt, run.maxAttempts); await githubStep(run, step, async () => { - await tagStep(run, { kind: "check" }); - for (const { jobPath, name } of read()) { await sink.update?.({ run, @@ -521,23 +476,18 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { const message = errorMessage(error).split("\n")[0]; - await githubStep( - run, - { - id: `github › check:${jobPath}:attempt:${attempt}`, - name: traceName.report(jobPath, traceName.retrying(attempt + 1, of)), - }, - async () => { - await sink.update?.({ - run, - name: jobCheckName(run, jobPath), - title: `Attempt ${attempt} of ${of}: ${message}`, - ...idFor(run, jobPath), - }); + const step = steps.failedAttempt(jobPath, attempt, of); - return null; - }, - ); + await githubStep(run, step, async () => { + await sink.update?.({ + run, + name: jobCheckName(run, jobPath), + title: `Attempt ${attempt} of ${of}: ${message}`, + ...idFor(run, jobPath), + }); + + return null; + }); }, target: ({ run, jobPath, name }) => { @@ -559,10 +509,8 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { } const { parent } = run.build; - const step = { - id: `github › check:${parent.jobPath}:building`, - name: traceName.report(parent.jobPath, "building"), - }; + + const step = steps.buildingCheck(parent.jobPath); await githubStep(run, step, async () => { await sink.update?.({ diff --git a/packages/ci/src/github/helpers.ts b/packages/ci/src/github/helpers.ts index 1e6abc3e9..0d3ed4190 100644 --- a/packages/ci/src/github/helpers.ts +++ b/packages/ci/src/github/helpers.ts @@ -8,7 +8,8 @@ import { RetryAfterError } from "inngest"; import { CiUsageError } from "../errors.ts"; import { durablePath } from "../pipeline/durable.ts"; -import { traceName } from "../pipeline/names.ts"; +import { ciRun } from "../pipeline/metadata.ts"; +import { steps, traceName } from "../pipeline/names.ts"; import type { CiRunScope } from "../pipeline/scope.ts"; import { countApi, @@ -40,7 +41,7 @@ const helperStep = async ( key: string, fn: () => Promise, /** The step's name, which is the helper's method name by default. */ - name = traceName.githubHelper(helper), + name?: string, ): Promise => { const run = requireRunScope(`github.${helper}`); @@ -54,7 +55,7 @@ const helperStep = async ( const id = `${job ? `${job.path} › ` : ""}github › ${helper}:${key}`; return inHelperSpan(run, () => { - return run.step.run({ id, name }, fn) as Promise; + return ciRun(run, steps.githubHelper(id, helper, name), fn); }); }; diff --git a/packages/ci/src/machine/admin.test.ts b/packages/ci/src/machine/admin.test.ts new file mode 100644 index 000000000..0f53433d8 --- /dev/null +++ b/packages/ci/src/machine/admin.test.ts @@ -0,0 +1,70 @@ +/** + * Tests of the Sandboxes API helpers cleanup shares: paged lists read as one + * stream, and snapshot deletes that tolerate what is already gone. + * + * @module + */ + +import type { Inngest } from "inngest"; +import { describe, expect, test, vi } from "vitest"; +import { deleteSnapshots, pages } from "./admin.ts"; + +const notFound = Object.assign(new Error("gone"), { + code: "sandbox_snapshot_not_found", +}); + +describe("pages", () => { + test("reads every page in order, passing the cursor on", async () => { + const list = vi.fn(async (cursor?: string) => { + return cursor === "next" + ? { items: [3], page: { hasMore: false } } + : { items: [1, 2], page: { hasMore: true, cursor: "next" } }; + }); + + const read: number[] = []; + + for await (const item of pages(list)) { + read.push(item); + } + + expect(read).toEqual([1, 2, 3]); + expect(list.mock.calls).toEqual([[undefined], ["next"]]); + }); +}); + +describe("deleteSnapshots", () => { + test("counts what it deletes, skips what is gone, and reports what fails", async () => { + const deleted: string[] = []; + + const get = async (id: string) => { + if (id === "gone") { + throw notFound; + } + + if (id === "stuck") { + throw new Error("refused"); + } + + return { + delete: async () => { + deleted.push(id); + }, + }; + }; + + const client = { + sandboxes: { snapshots: { get } }, + } as unknown as Inngest.Any; + const onFailure = vi.fn(); + + const result = await deleteSnapshots( + client, + ["a", "gone", "stuck", "b"], + onFailure, + ); + + expect(result).toEqual({ deleted: ["a", "b"], failed: ["stuck"] }); + expect(deleted).toEqual(["a", "b"]); + expect(onFailure).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/ci/src/machine/admin.ts b/packages/ci/src/machine/admin.ts new file mode 100644 index 000000000..c5dc75129 --- /dev/null +++ b/packages/ci/src/machine/admin.ts @@ -0,0 +1,75 @@ +/** + * Listing and deleting what CI's cleanup finds in the Sandboxes API: paged + * lists read as one stream, and snapshot deletes that tolerate what is already + * gone. Cleanup steps, run cleanup and invalidation all go through these + * rather than writing the paging and not-found handling again. + * + * @module + */ + +import type { Inngest } from "inngest"; +import { isSnapshotNotFound } from "../util.ts"; + +/** A page of a list the Sandboxes API returns. */ +export interface Page { + items: T[]; + page: { hasMore: boolean; cursor?: string }; +} + +/** Every item of a paged list, in order, fetching the next page as it is read. */ +export async function* pages( + list: (cursor?: string) => Promise>, +): AsyncGenerator { + let cursor: string | undefined; + + do { + const { items, page } = await list(cursor); + + yield* items; + + cursor = page.hasMore ? page.cursor : undefined; + } while (cursor); +} + +/** Every sandbox, whoever made it. */ +export const listSandboxes = (client: Inngest.Any) => { + return pages((cursor) => { + return client.sandboxes.list({ ...(cursor ? { cursor } : {}), limit: 100 }); + }); +}; + +/** + * Delete snapshots by ID. One that is already gone counts for nothing, and one + * that can't be deleted is reported to `onFailure` and left to expire, so a + * snapshot that stays never stops the rest. + */ +export const deleteSnapshots = async ( + client: Inngest.Any, + ids: Iterable, + onFailure?: (id: string, error: unknown) => void, +): Promise<{ deleted: string[]; failed: string[] }> => { + const deleted: string[] = []; + const failed: string[] = []; + + for (const id of ids) { + try { + const snapshot = await client.sandboxes.snapshots.get(id); + + if (snapshot) { + await snapshot.delete(); + + deleted.push(id); + } + } catch (error) { + if (isSnapshotNotFound(error)) { + continue; + } + + failed.push(id); + + onFailure?.(id, error); + } + } + + return { deleted, failed }; +}; diff --git a/packages/ci/src/machine/command.ts b/packages/ci/src/machine/command.ts index 7cf420bee..e8579835e 100644 --- a/packages/ci/src/machine/command.ts +++ b/packages/ci/src/machine/command.ts @@ -13,7 +13,8 @@ import { CommandFailedError, CommandTimeoutError, } from "../errors.ts"; -import { ciStep, spans, traceName } from "../pipeline/names.ts"; +import { ciRun } from "../pipeline/metadata.ts"; +import { ciStep, spans, steps, traceName } from "../pipeline/names.ts"; import type { CiJobScope, MachineHandle } from "../pipeline/scope.ts"; import { countApi, @@ -601,12 +602,9 @@ const recordFailure = async ( exitCode: number, ): Promise => { try { - await scope.run.step.run( - subStep(stepId, "exit", traceName.exited(exitCode)), - () => { - throw new NonRetriableError(`exit ${exitCode}`); - }, - ); + await ciRun(scope.run, steps.recordCommandFailure(stepId, exitCode), () => { + throw new NonRetriableError(`exit ${exitCode}`); + }); } catch (error) { if (!(error instanceof StepError)) { throw error; diff --git a/packages/ci/src/machine/machine.ts b/packages/ci/src/machine/machine.ts index 9a36a46f1..0249a24d7 100644 --- a/packages/ci/src/machine/machine.ts +++ b/packages/ci/src/machine/machine.ts @@ -9,7 +9,8 @@ import type { Inngest } from "inngest"; import type { CachedSnapshot, CacheTarget } from "../cache/cache.ts"; import { resolveTakenName, snapshotState } from "../cache/cache.ts"; import { CiUsageError } from "../errors.ts"; -import { ciStep, spans, traceName } from "../pipeline/names.ts"; +import { ciRun } from "../pipeline/metadata.ts"; +import { spans, steps, traceName } from "../pipeline/names.ts"; import type { CiJobScope, CiRunScope, @@ -25,13 +26,8 @@ import { } from "../pipeline/scope.ts"; import { inSpan } from "../pipeline/spans.ts"; import type { MachineConfig } from "../types.ts"; -import { - boundedName, - errorMessage, - isSandboxNotFound, - isSnapshotNotFound, - slug, -} from "../util.ts"; +import { boundedName, errorMessage, isSandboxNotFound, slug } from "../util.ts"; +import { deleteSnapshots, listSandboxes } from "./admin.ts"; /** * Create this scope's machine if it doesn't have one yet. @@ -359,35 +355,22 @@ const discardFailedStart = async ( ?.sandboxId; try { - await run.step.run( - { - id: joinId(stepId, "discard"), - name: traceName.discardMachine, - }, + await ciRun( + run, + steps.discardMachine(stepId), async (): Promise<{ id?: string }> => { // Errors are swallowed inside the step so it never retries. let id = knownId; try { - let cursor: string | undefined; - - while (!id) { - const page = await run.ci.client.sandboxes.list({ - ...(cursor ? { cursor } : {}), - limit: 100, - }); - - id = page.items.find( - (sandbox: { name: string; status: string }) => { - return sandbox.name === name && sandbox.status !== "TERMINATED"; - }, - )?.id; - - if (id || !page.page.hasMore) { - break; - } + if (!id) { + for await (const sandbox of listSandboxes(run.ci.client)) { + if (sandbox.name === name && sandbox.status !== "TERMINATED") { + id = sandbox.id; - cursor = page.page.cursor; + break; + } + } } if (id) { @@ -738,15 +721,9 @@ export const destroyRunMachines = async ( run: CiRunScope, attempt = 0, ): Promise => { - await run.step.run( - ciStep( - `pipeline${scopeSeparator}cleanup${attempt > 0 ? ` (attempt ${attempt})` : ""}`, - traceName.cleanUpMachines, - ), - async () => { - return destroyOrphans(run.ci.client, run.runId); - }, - ); + await ciRun(run, steps.cleanUpMachines(attempt), () => { + return destroyOrphans(run.ci.client, run.runId); + }); }; /** @@ -771,41 +748,14 @@ export const deleteRunSnapshots = async ( // Always there, and reads the set when it runs, for the reason cleaning up // machines is. - await run.step.run( - ciStep( - `pipeline${scopeSeparator}cleanup:snapshots${attempt > 0 ? ` (attempt ${attempt})` : ""}`, - traceName.cleanUpSnapshots, - ), - async () => { - const deleted: string[] = []; - const failed: string[] = []; - - for (const id of [...run.createdSnapshots]) { - try { - const snapshot = await run.ci.client.sandboxes.snapshots.get(id); - - if (snapshot) { - await snapshot.delete(); - - deleted.push(id); - } - } catch (error) { - if (isSnapshotNotFound(error)) { - continue; - } - - failed.push(id); - - run.ci.logger?.warn( - { snapshotId: id, error }, - "Couldn't delete a snapshot this run took; it will expire on its own", - ); - } - } - - return { deleted, failed }; - }, - ); + await ciRun(run, steps.cleanUpSnapshots(attempt), () => { + return deleteSnapshots(run.ci.client, run.createdSnapshots, (id, error) => { + run.ci.logger?.warn( + { snapshotId: id, error }, + "Couldn't delete a snapshot this run took; it will expire on its own", + ); + }); + }); }; /** @@ -818,32 +768,22 @@ export const destroyOrphans = async ( runId: string, ): Promise<{ destroyed: number }> => { const prefix = `ci-${runId}-`; - let cursor: string | undefined; let destroyed = 0; - do { - const page = await client.sandboxes.list({ - ...(cursor ? { cursor } : {}), - limit: 100, - }); - - for (const sandbox of page.items) { - if (sandbox.name.startsWith(prefix)) { - try { - await sandbox.destroy(); + for await (const sandbox of listSandboxes(client)) { + if (sandbox.name.startsWith(prefix)) { + try { + await sandbox.destroy(); - destroyed++; - } catch (error) { - // Anything but "not found" fails the step so it retries. - if (!isSandboxNotFound(error)) { - throw error; - } + destroyed++; + } catch (error) { + // Anything but "not found" fails the step so it retries. + if (!isSandboxNotFound(error)) { + throw error; } } } - - cursor = page.page.hasMore ? page.page.cursor : undefined; - } while (cursor); + } return { destroyed }; }; diff --git a/packages/ci/src/pipeline/durable.ts b/packages/ci/src/pipeline/durable.ts index 757ef95fa..7d4b08032 100644 --- a/packages/ci/src/pipeline/durable.ts +++ b/packages/ci/src/pipeline/durable.ts @@ -7,6 +7,7 @@ import { getAsyncCtx } from "inngest/experimental"; import { CiUsageError } from "../errors.ts"; +import { stepOptions, steps } from "./names.ts"; import { getJobScope, getRunScope, nextStepId } from "./scope.ts"; export type DurableBehaviour = "step" | "direct"; @@ -179,7 +180,10 @@ const call = async ( const step = getRunScope()?.step ?? execution.ctx.step; // Named by the method called, without the job path or count in its ID. - return step.run({ id, name: overrides.name ?? label }, invoke); + return step.run( + stepOptions(steps.githubCall(id, label, overrides.name)), + invoke, + ); }; const invokeOnClient = async ( diff --git a/packages/ci/src/pipeline/job.ts b/packages/ci/src/pipeline/job.ts index e482b6d8d..18acc8bb3 100644 --- a/packages/ci/src/pipeline/job.ts +++ b/packages/ci/src/pipeline/job.ts @@ -32,8 +32,8 @@ import { snapshotMachine } from "../machine/machine.ts"; import type { AnyJob, CheckConclusion, JobConfig } from "../types.ts"; import { errorMessage, formatDuration, shortReason } from "../util.ts"; import type { CacheBuildData, CacheBuildResult } from "./cacheBuild.ts"; -import { tagStep } from "./metadata.ts"; -import { ciStep, traceName } from "./names.ts"; +import { ciRun } from "./metadata.ts"; +import { ciStep, steps, traceName } from "./names.ts"; import type { CiJobScope, CiRunScope } from "./scope.ts"; import { getRunScope, @@ -545,12 +545,7 @@ const jobSteps = async ({ const checkStartedAt = checked ? await checks.jobStart(target) : undefined; const startedAt = checkStartedAt ?? - (await durableNow( - run, - `start:${scope.path}`, - traceName.recordStartTime, - scope.path, - )); + (await durableNow(run, `start:${scope.path}`, scope.path, "started")); if (checked) { run.openChecks.set(scope.path, checkName); @@ -629,12 +624,7 @@ const jobSteps = async ({ const endedAt = checkEndedAt ?? - (await durableNow( - run, - `end:${scope.path}`, - traceName.recordEndTime, - scope.path, - )); + (await durableNow(run, `end:${scope.path}`, scope.path, "ended")); const durationMs = endedAt - startedAt; run.summaries.push({ @@ -679,12 +669,7 @@ const jobSteps = async ({ const endedAt = checkEndedAt ?? - (await durableNow( - run, - `end:${scope.path}`, - traceName.recordEndTime, - scope.path, - )); + (await durableNow(run, `end:${scope.path}`, scope.path, "ended")); run.summaries.push({ path: scope.path, @@ -705,12 +690,10 @@ const jobSteps = async ({ const durableNow = ( run: CiRunScope, id: string, - name: string, jobPath: string, + edge: "started" | "ended", ): Promise => { - return run.step.run(ciStep(id, name), async () => { - await tagStep(run, { kind: "job", job: jobPath }); - + return ciRun(run, steps.recordTime(id, jobPath, edge), () => { return Date.now(); }); }; diff --git a/packages/ci/src/pipeline/metadata.ts b/packages/ci/src/pipeline/metadata.ts index 72f2c1657..7672e3284 100644 --- a/packages/ci/src/pipeline/metadata.ts +++ b/packages/ci/src/pipeline/metadata.ts @@ -11,7 +11,7 @@ import { getAsyncCtx } from "inngest/experimental"; import type { CheckConclusion } from "../types.ts"; import { version } from "../version.ts"; -import { ciStep, traceName } from "./names.ts"; +import { type StepSpec, stepOptions, steps } from "./names.ts"; import type { CiRunScope } from "./scope.ts"; import { apiNames } from "./scope.ts"; @@ -141,6 +141,25 @@ export const tagStep = async ( } }; +/** + * Run `fn` as one of CI's own steps, from the `steps` catalog. The step is + * tagged with what its spec says it is for, and with `runValues` on the run. + */ +export const ciRun = ( + run: CiRunScope, + spec: StepSpec, + fn: () => T | Promise, + runValues?: () => Record, +): Promise => { + return run.step.run(stepOptions(spec), async () => { + if (spec.tag || runValues) { + await tagStep(run, spec.tag, runValues?.()); + } + + return fn(); + }) as Promise; +}; + /** * A step of its own that only carries run metadata and returns the time, for * a pipeline whose check is off and so has no step to ride on. @@ -150,9 +169,12 @@ export const metadataStep = ( id: string, runValues: () => Record, ): Promise => { - return run.step.run(ciStep(id, traceName.recordRunDetails), async () => { - await tagStep(run, undefined, runValues()); - - return Date.now(); - }); + return ciRun( + run, + steps.recordRunDetails(id), + () => { + return Date.now(); + }, + runValues, + ); }; diff --git a/packages/ci/src/pipeline/names.ts b/packages/ci/src/pipeline/names.ts index 475a259b5..a58ab1868 100644 --- a/packages/ci/src/pipeline/names.ts +++ b/packages/ci/src/pipeline/names.ts @@ -13,6 +13,7 @@ import type { CheckConclusion } from "../types.ts"; import { formatDuration } from "../util.ts"; import { version } from "../version.ts"; +import type { StepTag } from "./metadata.ts"; import type { CiRunScope } from "./scope.ts"; import { joinId, scopeSeparator } from "./scope.ts"; import type { SpanInfo, SpanKind } from "./spans.ts"; @@ -94,6 +95,21 @@ export const spans = { }, }; +/** What one of CI's own steps is: its ID, name, and what it's marked with. */ +export interface StepSpec { + /** The step's ID, which replays depend on. */ + id: string; + /** What the step is called in the trace. */ + name: string; + /** What the step is for, when it belongs to a job, a check or the cache. */ + tag?: StepTag; + /** + * Set for a step that CI's origin shouldn't mark: something you called, or a + * step inside a span that already carries it. + */ + unmarked?: true; +} + /** * A step CI runs for you where no span of CI's marks it, such as a command's * `Start process` inside the command's span, which is yours. @@ -102,6 +118,13 @@ export const ciStep = (id: string, name: string) => { return { id, name, ...originOption(ciOrigin) }; }; +/** The step options for a spec, marked as CI's work unless it says not. */ +export const stepOptions = (spec: StepSpec) => { + return spec.unmarked + ? { id: spec.id, name: spec.name } + : ciStep(spec.id, spec.name); +}; + /** How a finished check reads, as in `Report test: passed`. */ const outcomes: Partial> = { success: "passed", @@ -109,6 +132,233 @@ const outcomes: Partial> = { timed_out: "timed out", }; +/** A check update's name, as in `Report test: started`. */ +const report = (subject: string, status: string): string => { + return `Report ${subject}: ${status}`; +}; + +/** A retry's name, as in `retrying (attempt 2 of 3)`. */ +const retrying = (attempt: number, of: number): string => { + return `retrying (attempt ${attempt} of ${of})`; +}; + +const cacheTag = (job: string): StepTag => { + return { kind: "cache", job }; +}; + +const checkTag = (job?: string): StepTag => { + return job === undefined ? { kind: "check" } : { kind: "check", job }; +}; + +/** The ID of a step of a cleanup, which a retried cleanup repeats. */ +const cleanupId = (what: string, attempt: number): string => { + return joinId( + "pipeline", + `${what}${attempt > 0 ? ` (attempt ${attempt})` : ""}`, + ); +}; + +/** The rest of a spec, once its ID and name are known. */ +type StepExtras = Omit; + +/** A step with a fixed name, whose ID the caller makes. */ +const named = (name: string, extras: StepExtras = {}) => { + return (id: string): StepSpec => { + return { id, name, ...extras }; + }; +}; + +/** A step with a fixed ID and name. */ +const fixed = (id: string, name: string, extras: StepExtras = {}) => { + return (): StepSpec => { + return { id, name, ...extras }; + }; +}; + +/** + * Every step CI runs for you: its ID, name and tag, in one place. Adding a + * step or renaming one is an edit to its entry here, and the step itself is + * `ciRun(run, steps.x(...), fn)`. Steps that CI's origin shouldn't mark are + * `unmarked`. + */ +export const steps = { + // The cache. + cacheKey: (jobPath: string): StepSpec => { + return { + id: joinId(jobPath, "cache:key"), + name: "Check cache", + tag: cacheTag(jobPath), + }; + }, + + lookUpCache: (id: string, jobPath: string): StepSpec => { + return { id, name: "Look up cache", tag: cacheTag(jobPath) }; + }, + + resolveCacheName: named("Resolve cache name"), + checkSnapshotState: named("Check snapshot state"), + deleteBadSnapshot: named("Delete bad snapshot"), + + // Sandboxes. + discardMachine: (stepId: string): StepSpec => { + return { + id: joinId(stepId, "discard"), + name: "Discard sandbox", + unmarked: true, + }; + }, + + recordCommandFailure: (stepId: string, exitCode: number): StepSpec => { + return { + id: joinId(stepId, "exit"), + name: `Exited with code ${exitCode}`, + }; + }, + + cleanUpMachines: (attempt: number): StepSpec => { + return { id: cleanupId("cleanup", attempt), name: "Clean up sandboxes" }; + }, + + cleanUpEndedRun: fixed("destroy-orphans", "Clean up sandboxes"), + + cleanUpSnapshots: (attempt: number): StepSpec => { + return { + id: cleanupId("cleanup:snapshots", attempt), + name: "Clean up snapshots", + }; + }, + + // Checking out. + uploadWorkingTree: named("Upload working tree", { unmarked: true }), + cloneRepository: named("Clone repository", { unmarked: true }), + findChangedFiles: fixed("changed", "Find changed files", { unmarked: true }), + + // What you called on a run. + githubHelper: (id: string, helper: string, name = `github.${helper}`) => { + return { id, name, unmarked: true } satisfies StepSpec; + }, + + githubCall: (id: string, label: string, name = label) => { + return { id, name, unmarked: true } satisfies StepSpec; + }, + + addSummary: named("Add summary", { unmarked: true }), + addAnnotations: named("Add annotations", { unmarked: true }), + + // Reading from GitHub. + resolveRepository: (): StepSpec => { + return { + id: joinId("github", "repo:resolve"), + name: "Resolve repository", + unmarked: true, + }; + }, + + resolvePullRequest: (): StepSpec => { + return { + id: joinId("github", "pr:resolve"), + name: "pr:resolve", + unmarked: true, + }; + }, + + commentNotAllowed: (): StepSpec => { + return { + id: joinId("github", "comment:denied"), + name: "Comment: not allowed", + unmarked: true, + }; + }, + + // Checks. + startCheck: (check: string, job?: string): StepSpec => { + return { + id: joinId("github", `check:${job ?? check}:start`), + name: + job === undefined ? `Create check: ${check}` : report(job, "started"), + tag: checkTag(job), + unmarked: true, + }; + }, + + completeCheck: ( + check: string, + job?: { path: string; conclusion: CheckConclusion }, + ): StepSpec => { + return { + id: joinId("github", `check:${job?.path ?? check}:complete`), + name: job + ? report(job.path, outcomes[job.conclusion] ?? job.conclusion) + : `Complete check: ${check}`, + tag: checkTag(job?.path), + unmarked: true, + }; + }, + + completeJobChecks: (): StepSpec => { + return { + id: joinId("github", "check:jobs:complete"), + name: report("jobs", "ended with the run"), + tag: checkTag(), + unmarked: true, + }; + }, + + retryCheck: ( + check: string, + attempt: number, + of: number, + job?: string, + ): StepSpec => { + return { + id: joinId("github", `check:${job ?? "pipeline"}:retry:${attempt}`), + name: report(job ?? check, retrying(attempt + 2, of)), + tag: checkTag(job), + unmarked: true, + }; + }, + + retryJobChecks: (attempt: number, of: number): StepSpec => { + return { + id: joinId("github", `check:jobs:retry:${attempt}`), + name: report("jobs", retrying(attempt + 2, of)), + tag: checkTag(), + unmarked: true, + }; + }, + + failedAttempt: (job: string, attempt: number, of: number): StepSpec => { + return { + id: joinId("github", `check:${job}:attempt:${attempt}`), + name: report(job, retrying(attempt + 1, of)), + unmarked: true, + }; + }, + + buildingCheck: (job: string): StepSpec => { + return { + id: joinId("github", `check:${job}:building`), + name: report(job, "building"), + unmarked: true, + }; + }, + + // Times and run details, for steps that carry nothing else. + recordTime: ( + id: string, + job: string, + edge: "started" | "ended", + ): StepSpec => { + return { + id, + name: edge === "started" ? "Record start time" : "Record end time", + tag: { kind: "job", job }, + }; + }, + + recordRunDetails: named("Record run details"), +}; + export const traceName = { /** A job's span: its `name`, or its path when it has none. */ job: (run: CiRunScope, path: string): string => { @@ -138,40 +388,10 @@ export const traceName = { return `Wait ${formatDuration(ms)}`; }, - exited: (exitCode: number): string => { - return `Exited with code ${exitCode}`; - }, - buildInOwnRun: (path: string): string => { return `Build ${path} in its own run`; }, - createCheck: (check: string): string => { - return `Create check: ${check}`; - }, - - completeCheck: (check: string): string => { - return `Complete check: ${check}`; - }, - - /** A check update, as in `Report test: started`. */ - report: (subject: string, status: string): string => { - return `Report ${subject}: ${status}`; - }, - - retrying: (attempt: number, of: number): string => { - return `retrying (attempt ${attempt} of ${of})`; - }, - - outcome: (conclusion: CheckConclusion): string => { - return outcomes[conclusion] ?? conclusion; - }, - - /** A `github.*` helper you called, by its method name. */ - githubHelper: (helper: string): string => { - return `github.${helper}`; - }, - canUser: (login: string, permission: string): string => { return `Check ${login} can ${permission}`; }, @@ -185,19 +405,14 @@ export const traceName = { }, github: "GitHub", - resolveRepository: "Resolve repository", - commentNotAllowed: "Comment: not allowed", createMachine: "Create sandbox", restartMachine: "Restart sandbox", createFreshMachine: "Create sandbox (fresh)", retryCreate: "Retry create", - discardMachine: "Discard sandbox", prepareWorkspace: "Prepare workspace", saveMachine: "Save sandbox", snapshotMachine: "Snapshot sandbox", - cleanUpMachines: "Clean up sandboxes", - cleanUpSnapshots: "Clean up snapshots", /** A captured command's one step: it runs and returns its output at once. */ runAndReadOutput: "Run and read output", @@ -207,20 +422,4 @@ export const traceName = { stopProcess: "Stop process", stopAfterTimeout: "Stop after timeout", findStartedProcess: "Find started process", - - checkCache: "Check cache", - lookUpCache: "Look up cache", - resolveCacheName: "Resolve cache name", - deleteBadSnapshot: "Delete bad snapshot", - checkSnapshotState: "Check snapshot state", - - cloneRepository: "Clone repository", - uploadWorkingTree: "Upload working tree", - findChangedFiles: "Find changed files", - - recordStartTime: "Record start time", - recordEndTime: "Record end time", - recordRunDetails: "Record run details", - addSummary: "Add summary", - addAnnotations: "Add annotations", }; diff --git a/packages/ci/src/pipeline/pipeline.ts b/packages/ci/src/pipeline/pipeline.ts index 4b5c9ce63..8c8eb2c9e 100644 --- a/packages/ci/src/pipeline/pipeline.ts +++ b/packages/ci/src/pipeline/pipeline.ts @@ -48,8 +48,13 @@ import { formatDuration } from "../util.ts"; import type { CacheBuildData } from "./cacheBuild.ts"; import type { RegisteredJob } from "./job.ts"; import { conclusionForError, runJob } from "./job.ts"; -import { metadataStep, runEndMetadata, runStartMetadata } from "./metadata.ts"; -import { ciStep, traceName } from "./names.ts"; +import { + ciRun, + metadataStep, + runEndMetadata, + runStartMetadata, +} from "./metadata.ts"; +import { stepOptions, steps } from "./names.ts"; import type { CiInternals, CiRunScope } from "./scope.ts"; import { apiNames, @@ -569,8 +574,9 @@ const resolveConfiguredRepo = ( const { owner, name } = parseRepo(fullName); return inGitHubSpan(run, () => { - return run.step.run( - { id: `github › repo:resolve`, name: traceName.resolveRepository }, + return ciRun( + run, + steps.resolveRepository(), async (): Promise => { const base: RepoContext = { owner, name, fullName, sha: "" }; const provider = run.ci.github as GitHubProvider; @@ -601,7 +607,7 @@ const resolveConfiguredRepo = ( (run.event as { name?: string } | undefined)?.name ?? "manual", }; }, - ) as Promise; + ); }); }; @@ -616,8 +622,9 @@ const resolvePullRequestHead = ( run: CiRunScope, repo: RepoContext, ): Promise => { - return run.step.run( - { id: `github › pr:resolve`, name: "pr:resolve" }, + return ciRun( + run, + steps.resolvePullRequest(), async (): Promise => { const provider = run.ci.github as GitHubProvider; const number = repo.pullRequest?.number; @@ -649,7 +656,7 @@ const resolvePullRequestHead = ( }, }; }, - ) as Promise; + ); }; /** Errors that replaying the run would only reproduce. */ @@ -832,19 +839,16 @@ const checkCommentPermission = async ( if (!allowed) { await inGitHubSpan(run, () => { - return run.step.run( - { id: "github › comment:denied", name: traceName.commentNotAllowed }, - async () => { - const { stickyComment } = await import("../github/helpers.ts"); + return ciRun(run, steps.commentNotAllowed(), async () => { + const { stickyComment } = await import("../github/helpers.ts"); - await stickyComment( - "permission", - `@${login} you need \`${minPermission}\` permission to run \`${commentBody(run.event as { data?: unknown }).split(" ")[0]}\`.`, - ); + await stickyComment( + "permission", + `@${login} you need \`${minPermission}\` permission to run \`${commentBody(run.event as { data?: unknown }).split(" ")[0]}\`.`, + ); - return { denied: login }; - }, - ); + return { denied: login }; + }); }); } @@ -899,12 +903,9 @@ export const cleanupFunction = ({ async ({ event, step }: any) => { const runId = event?.data?.run_id ?? event?.data?.runId; - return step.run( - ciStep("destroy-orphans", traceName.cleanUpMachines), - async () => { - return runId ? destroyOrphans(client, runId) : { destroyed: 0 }; - }, - ); + return step.run(stepOptions(steps.cleanUpEndedRun()), () => { + return runId ? destroyOrphans(client, runId) : { destroyed: 0 }; + }); }, ); }; diff --git a/packages/ci/src/pipeline/stepNames.test.ts b/packages/ci/src/pipeline/stepNames.test.ts new file mode 100644 index 000000000..7c5dd178c --- /dev/null +++ b/packages/ci/src/pipeline/stepNames.test.ts @@ -0,0 +1,66 @@ +/** + * Tests that the step catalog keeps the IDs and names the trace and replays + * already depend on, and marks as CI's work only the steps that were marked. + * + * @module + */ + +import { describe, expect, test } from "vitest"; +import { ciOrigin, stepOptions, steps } from "./names.ts"; + +describe("the step catalog", () => { + test.each([ + [steps.cacheKey("test"), "test › cache:key", "Check cache"], + [steps.cleanUpMachines(0), "pipeline › cleanup", "Clean up sandboxes"], + [ + steps.cleanUpSnapshots(2), + "pipeline › cleanup:snapshots (attempt 2)", + "Clean up snapshots", + ], + [steps.cleanUpEndedRun(), "destroy-orphans", "Clean up sandboxes"], + [ + steps.recordCommandFailure("test › 1", 2), + "test › 1 › exit", + "Exited with code 2", + ], + [steps.startCheck("CI"), "github › check:CI:start", "Create check: CI"], + [ + steps.startCheck("CI / test", "test"), + "github › check:test:start", + "Report test: started", + ], + [ + steps.completeCheck("CI / test", { path: "test", conclusion: "failure" }), + "github › check:test:complete", + "Report test: failed", + ], + [ + steps.retryCheck("CI", 0, 3), + "github › check:pipeline:retry:0", + "Report CI: retrying (attempt 2 of 3)", + ], + [ + steps.failedAttempt("test", 1, 3), + "github › check:test:attempt:1", + "Report test: retrying (attempt 2 of 3)", + ], + [steps.githubHelper("id", "comment"), "id", "github.comment"], + [ + steps.recordTime("start:test", "test", "ended"), + "start:test", + "Record end time", + ], + ])("%#", (spec, id, name) => { + expect({ id: spec.id, name: spec.name }).toEqual({ id, name }); + }); + + test("marks CI's own work with its origin, and an unmarked step without", () => { + expect(stepOptions(steps.cacheKey("test"))).toMatchObject({ + "~origin": ciOrigin, + }); + expect(stepOptions(steps.findChangedFiles())).toEqual({ + id: "changed", + name: "Find changed files", + }); + }); +}); diff --git a/packages/ci/src/report.ts b/packages/ci/src/report.ts index c66c13230..d22675699 100644 --- a/packages/ci/src/report.ts +++ b/packages/ci/src/report.ts @@ -4,7 +4,8 @@ * @module */ -import { traceName } from "./pipeline/names.ts"; +import { ciRun } from "./pipeline/metadata.ts"; +import { steps } from "./pipeline/names.ts"; import { countApi, getJobScope, @@ -44,7 +45,7 @@ export const report = { const id = nextStepId(run, job?.path, "report:summary"); - await run.step.run({ id, name: traceName.addSummary }, () => { + await ciRun(run, steps.addSummary(id), () => { return { length: markdown.length, }; @@ -95,7 +96,7 @@ export const report = { const id = nextStepId(run, job?.path, "report:annotate"); - await run.step.run({ id, name: traceName.addAnnotations }, () => { + await ciRun(run, steps.addAnnotations(id), () => { return { count: valid.length, }; diff --git a/packages/ci/src/units.test.ts b/packages/ci/src/units.test.ts index c15e89fe0..d5aefad5a 100644 --- a/packages/ci/src/units.test.ts +++ b/packages/ci/src/units.test.ts @@ -6,7 +6,8 @@ */ import { describe, expect, test } from "vitest"; -import { cacheScopes, lookupCache, snapshotName } from "./cache/cache.ts"; +import { cacheScopes, lookupCache } from "./cache/cache.ts"; +import { formatName } from "./cache/names.ts"; import { CiUsageError } from "./errors.ts"; import { batchAnnotations, @@ -684,7 +685,12 @@ describe("formatting", () => { }); describe("looking a cached snapshot up by name", () => { - const name = snapshotName("global", "setup", "k1"); + const name = formatName({ + kind: "cache", + scope: "global", + jobId: "setup", + ownKey: "k1", + }); // One moment for every fixture, so two snapshots built apart still match. const now = Date.now(); @@ -805,12 +811,29 @@ describe("looking a cached snapshot up by name", () => { describe("snapshot names", () => { test("say what they are for", () => { - expect(snapshotName("pr:4", "setup", "abc")).toBe("ci/pr:4/setup/abc"); + expect( + formatName({ + kind: "cache", + scope: "pr:4", + jobId: "setup", + ownKey: "abc", + }), + ).toBe("ci/pr:4/setup/abc"); }); test("too long a name keeps its start and stays unique", () => { - const long = snapshotName("main", "j".repeat(300), "abc"); - const other = snapshotName("main", "j".repeat(300), "abd"); + const long = formatName({ + kind: "cache", + scope: "main", + jobId: "j".repeat(300), + ownKey: "abc", + }); + const other = formatName({ + kind: "cache", + scope: "main", + jobId: "j".repeat(300), + ownKey: "abd", + }); expect(long).toHaveLength(255); expect(long.startsWith("ci/main/jjj")).toBe(true); From aa6b844345675380d755a6dd3d719515572339ed Mon Sep 17 00:00:00 2001 From: x Date: Tue, 13 Oct 2026 17:14:44 +0000 Subject: [PATCH 06/11] feat(ci): add cache.maxAge A cached snapshot older than cache.maxAge counts as a miss. One usableFor predicate decides whether a snapshot may be used. Co-Authored-By: Claude Opus 5.5 --- .changeset/ci-cache-max-age.md | 5 ++ packages/ci/README.md | 14 ++++ packages/ci/src/cache/cache.test.ts | 71 +++++++++++++++++- packages/ci/src/cache/cache.ts | 72 +++++++++++++++--- packages/ci/src/machine/machine.ts | 7 +- .../ci/src/pipeline/cacheGuarantees.test.ts | 73 +++++++++++++++++++ packages/ci/src/pipeline/job.ts | 33 ++++++++- packages/ci/src/types.ts | 9 +++ 8 files changed, 266 insertions(+), 18 deletions(-) create mode 100644 .changeset/ci-cache-max-age.md diff --git a/.changeset/ci-cache-max-age.md b/.changeset/ci-cache-max-age.md new file mode 100644 index 000000000..3387a1b04 --- /dev/null +++ b/.changeset/ci-cache-max-age.md @@ -0,0 +1,5 @@ +--- +"@inngest/ci": minor +--- + +Add `cache.maxAge`: a cached snapshot older than it counts as a miss, so the job builds again and the new snapshot takes the name. A malformed `maxAge` throws when the job is defined. diff --git a/packages/ci/README.md b/packages/ci/README.md index bddaf9da9..10f8e040a 100644 --- a/packages/ci/README.md +++ b/packages/ci/README.md @@ -51,6 +51,20 @@ const test = ci.job({ id: "test", from: base }, async () => { A pipeline run is one trace. `lint` and `test` start from a snapshot of `base`, and a failed command runs again without rerunning the jobs that passed. +## Caching + +A job with a `cache` key is reused until its key changes. Add `maxAge` to rebuild it on a schedule too: a snapshot older than that counts as a miss, and the new one takes its name. + +```ts +const base = ci.job( + { id: "base", cache: { key: files("images/node-base/**"), maxAge: "1d" } }, + async () => { + await checkout(); + await $`pnpm install`; + }, +); +``` + ## Install ```bash diff --git a/packages/ci/src/cache/cache.test.ts b/packages/ci/src/cache/cache.test.ts index 5cc4673f5..79183c6ca 100644 --- a/packages/ci/src/cache/cache.test.ts +++ b/packages/ci/src/cache/cache.test.ts @@ -10,7 +10,7 @@ import { describe, expect, test } from "vitest"; import { deleteRunSnapshots } from "../machine/machine.ts"; import type { CiRunScope } from "../pipeline/scope.ts"; -import { deleteSnapshot, resolveTakenName } from "./cache.ts"; +import { deleteSnapshot, resolveTakenName, usableFor } from "./cache.ts"; /** Step results by ID, kept across the "replays" of one test. */ type Memo = Map; @@ -89,12 +89,16 @@ describe("deleteSnapshot", () => { describe("resolveTakenName", () => { /** A run whose only snapshot holds the name, recording what is deleted. */ - const holding = (deleted: string[]): CiRunScope => { + const holding = ( + deleted: string[], + /** When the holder was created, in place of now. */ + createdAt = new Date().toISOString(), + ): CiRunScope => { const holder = { id: "main-snapshot", name: "ci/main/base/k", status: "READY", - createdAt: new Date().toISOString(), + createdAt, }; const snapshots = { @@ -130,6 +134,7 @@ describe("resolveTakenName", () => { holding(deleted), "name-taken", "ci/main/base/k", + undefined, "main-snapshot", ); @@ -144,6 +149,7 @@ describe("resolveTakenName", () => { holding(deleted), "name-taken", "ci/main/base/k", + undefined, "main-snapshot", true, ); @@ -151,4 +157,63 @@ describe("resolveTakenName", () => { expect(deleted).toEqual(["main-snapshot"]); expect(taken.cleared).toBe(true); }); + + test("a holder that is too old is replaced without being known to be broken", async () => { + const deleted: string[] = []; + + const taken = await resolveTakenName( + holding(deleted, "2020-01-01T00:00:00.000Z"), + "name-taken", + "ci/main/base/k", + { key: "k", maxAge: "1m" }, + ); + + expect(deleted).toEqual(["main-snapshot"]); + expect(taken.cleared).toBe(true); + }); +}); + +describe("usableFor", () => { + const ago = (ms: number): string => { + return new Date(Date.now() - ms).toISOString(); + }; + + const soon = new Date(Date.now() + 60_000).toISOString(); + + test.each([ + ["ready and fresh", undefined, undefined, {}, true], + ["still creating", undefined, undefined, { status: "CREATING" }, false], + ["the excluded one", undefined, "s1", {}, false], + ["about to expire", undefined, undefined, { expiresAt: soon }, false], + ["younger than maxAge", "1h", undefined, { createdAt: ago(60_000) }, true], + [ + "older than maxAge", + "1h", + undefined, + { createdAt: ago(7_200_000) }, + false, + ], + ["an unreadable createdAt", "1h", undefined, { createdAt: "?" }, true], + [ + "old with no maxAge", + undefined, + undefined, + { createdAt: ago(1e10) }, + true, + ], + ] as const)("%s", (_name, maxAge, exclude, overrides, expected) => { + const usable = usableFor( + maxAge ? { key: "k", maxAge } : undefined, + exclude, + ); + + expect( + usable({ + id: "s1", + status: "READY", + createdAt: ago(0), + ...overrides, + }), + ).toBe(expected); + }); }); diff --git a/packages/ci/src/cache/cache.ts b/packages/ci/src/cache/cache.ts index 59f072bdd..1fa4957bf 100644 --- a/packages/ci/src/cache/cache.ts +++ b/packages/ci/src/cache/cache.ts @@ -23,6 +23,7 @@ import type { RepoContext, } from "../types.ts"; import { + durationToMs, formatRelative, hash, isSnapshotNotFound, @@ -328,6 +329,46 @@ const isExpiring = (expiresAt: string | undefined): boolean => { return Number.isFinite(at) && at - Date.now() < expiryMarginMs; }; +/** The parts of a snapshot that decide whether a job may use it. */ +export interface SnapshotInfo { + id: string; + status: string; + createdAt: string; + expiresAt?: string; +} + +/** Whether a job may use a snapshot. */ +export type Usable = (snapshot: SnapshotInfo) => boolean; + +/** + * What makes a snapshot usable: ready, not the one to avoid, not about to + * expire, and not older than the job's `cache.maxAge`. A new freshness rule is + * one more clause here. `defineJob` has already checked that `maxAge` parses. + */ +export const usableFor = ( + cache: CacheConfig | undefined, + /** A snapshot that must not be used, though it may still hold the name. */ + exclude?: string, +): Usable => { + const maxAgeMs = + cache?.maxAge === undefined ? undefined : durationToMs(cache.maxAge); + + return (snapshot) => { + const createdAt = Date.parse(snapshot.createdAt); + const tooOld = + maxAgeMs !== undefined && + Number.isFinite(createdAt) && + Date.now() - createdAt > maxAgeMs; + + return ( + snapshot.status === "READY" && + snapshot.id !== exclude && + !isExpiring(snapshot.expiresAt) && + !tooOld + ); + }; +}; + /** How long a lookup waits for a snapshot another build is still taking. */ const readyWaitMs = 2 * 60 * 1000; const readyPollMs = 1000; @@ -378,7 +419,8 @@ const waitUntilReady = async ( /** * The newest snapshot with exactly this name, if it can be used: ready, or - * ready after waiting for the build taking it, and not about to expire. + * ready after waiting for the build taking it, not about to expire, and not + * older than the job's max age. * * Any error is a miss, and only an exact name counts, since a server that * doesn't know names ignores the filter and lists every snapshot. @@ -386,8 +428,7 @@ const waitUntilReady = async ( const findNamed = async ( run: CiRunScope, name: string, - /** A snapshot that must not be used, though it may still hold the name. */ - exclude?: string, + usable: Usable, ): Promise => { try { const page = (await snapshotsClient(run).list({ name, limit: 10 })) as { @@ -398,7 +439,8 @@ const findNamed = async ( return snapshot.name === name; }); - if (!newest || newest.id === exclude) { + // Waiting is only worth it for a snapshot that would be usable once ready. + if (!newest || !usable({ ...newest, status: "READY" })) { return undefined; } @@ -407,7 +449,7 @@ const findNamed = async ( ? await waitUntilReady(run, newest.id) : newest; - if (ready?.status !== "READY" || isExpiring(ready.expiresAt)) { + if (!ready || !usable(ready)) { return undefined; } @@ -425,11 +467,13 @@ const findInScopes = async ( ownKey: string, exclude?: string, ): Promise => { + const usable = usableFor(cache, exclude); + for (const scope of cacheScopes(run.repo, cache.scope).read) { const found = await findNamed( run, formatName({ kind: "cache", scope, jobId, ownKey }), - exclude, + usable, ); if (found) { @@ -488,7 +532,7 @@ const findCached = ( ): Promise => { return cache ? findInScopes(run, jobId, cache, target.ownKey, exclude) - : findNamed(run, target.name, exclude); + : findNamed(run, target.name, usableFor(undefined, exclude)); }; /** @@ -524,13 +568,16 @@ export const lookupBeforeBuild = async ( /** * Find the snapshot that holds a name a build couldn't take, as a memoized - * step. If it can't be used, because it is about to expire or is the bad one - * the build replaces, it is deleted so the build can take the name after all. + * step. If it can't be used, because it is about to expire, is older than the + * job's max age or is the bad one the build replaces, it is deleted so the + * build can take the name after all. */ export const resolveTakenName = async ( run: CiRunScope, stepId: string, name: string, + /** The job's `cache`, whose `maxAge` a holder may be past. */ + cache?: CacheConfig, exclude?: string, /** * Whether `exclude` was decided to be broken, which is the only case where @@ -543,7 +590,7 @@ export const resolveTakenName = async ( run, steps.resolveCacheName(stepId), async (): Promise<{ winner?: CachedSnapshot; cleared: boolean }> => { - const winner = await findNamed(run, name, exclude); + const winner = await findNamed(run, name, usableFor(cache, exclude)); if (winner) { return { winner, cleared: false }; @@ -556,9 +603,10 @@ export const resolveTakenName = async ( items: SnapshotResource[]; }; + const usable = usableFor(cache); + for (const holder of page.items) { - const unusable = - (broken && holder.id === exclude) || isExpiring(holder.expiresAt); + const unusable = (broken && holder.id === exclude) || !usable(holder); if (holder.name !== name || holder.status !== "READY" || !unusable) { continue; diff --git a/packages/ci/src/machine/machine.ts b/packages/ci/src/machine/machine.ts index 0249a24d7..18a03381b 100644 --- a/packages/ci/src/machine/machine.ts +++ b/packages/ci/src/machine/machine.ts @@ -25,7 +25,7 @@ import { scopeSeparator, } from "../pipeline/scope.ts"; import { inSpan } from "../pipeline/spans.ts"; -import type { MachineConfig } from "../types.ts"; +import type { CacheConfig, MachineConfig } from "../types.ts"; import { boundedName, errorMessage, isSandboxNotFound, slug } from "../util.ts"; import { deleteSnapshots, listSandboxes } from "./admin.ts"; @@ -466,6 +466,8 @@ export interface SnapshotCache { * failing to name it isn't worth a warning. */ ephemeral?: boolean; + /** The job's `cache`, whose `maxAge` a snapshot holding the name may be past. */ + cache?: CacheConfig; } /** A snapshot of a job's machine. */ @@ -556,7 +558,7 @@ const createNamedSnapshot = async ( handle: MachineHandle, jobPath: string, stepId: string, - { target, exclude, broken, ephemeral }: SnapshotCache, + { target, exclude, broken, ephemeral, cache }: SnapshotCache, ): Promise => { const name = target.name; @@ -591,6 +593,7 @@ const createNamedSnapshot = async ( run, `${stepId}${scopeSeparator}name-taken`, name, + cache, exclude, broken, ); diff --git a/packages/ci/src/pipeline/cacheGuarantees.test.ts b/packages/ci/src/pipeline/cacheGuarantees.test.ts index c34aa0b88..f9d3a4ef6 100644 --- a/packages/ci/src/pipeline/cacheGuarantees.test.ts +++ b/packages/ci/src/pipeline/cacheGuarantees.test.ts @@ -503,6 +503,79 @@ describe("jobs that start from one parent", () => { }); }); +describe("a snapshot older than the cache's maxAge", () => { + const hours = 60 * 60 * 1000; + + /** Run a one-job pipeline, age its snapshot, and run it again. */ + const rerunAged = async (ageMs: number) => { + const api = createFakeSandboxApi(); + const { ci } = setup(api); + + const install = ci.job( + { id: "install", cache: { key: "v1", maxAge: "1d" } }, + async () => { + await $`pnpm install`; + }, + ); + + const pipeline = ci.pipeline({ id: "pr", on: prTrigger }, async () => { + await install(); + }); + + await runFunction(pipeline, { event: prEvent, runId: "01FIRST" }); + + const [holder] = named(api); + + (holder as { createdAt: string }).createdAt = new Date( + Date.now() - ageMs, + ).toISOString(); + + const second = await runFunction(pipeline, { + event: prEvent, + runId: "01SECOND", + }); + + expect(second.type).toBe("function-resolved"); + + return { api, holder }; + }; + + test("is reused while it is younger", async () => { + const { api, holder } = await rerunAged(hours); + + expect(count(api, "pnpm install")).toBe(1); + expect(api.snapshots.has(holder?.id ?? "")).toBe(true); + }); + + test("is rebuilt, and the rebuild takes the name", async () => { + const { api, holder } = await rerunAged(48 * hours); + + expect(count(api, "pnpm install")).toBe(2); + expect(api.snapshots.has(holder?.id ?? "")).toBe(false); + + const [replacement, ...rest] = named(api); + + expect(rest).toEqual([]); + expect(replacement?.id).not.toBe(holder?.id); + expect(replacement?.name).toBe(holder?.name); + expect(replacement?.status).toBe("READY"); + }); + + test.each(["a day", "0s"])( + "a maxAge of %j throws when the job is defined", + (maxAge) => { + const { ci } = setup(createFakeSandboxApi()); + + expect(() => { + return ci.job( + { id: "install", cache: { key: "v1", maxAge } }, + async () => {}, + ); + }).toThrow(/cache\.maxAge/); + }, + ); +}); + describe("a job that starts from a job in its own app", () => { /** * `test` starts from `build`, which starts from `install`, each keyed on diff --git a/packages/ci/src/pipeline/job.ts b/packages/ci/src/pipeline/job.ts index 18acc8bb3..4fc2ecbd4 100644 --- a/packages/ci/src/pipeline/job.ts +++ b/packages/ci/src/pipeline/job.ts @@ -30,7 +30,12 @@ import { } from "../machine/from.ts"; import { snapshotMachine } from "../machine/machine.ts"; import type { AnyJob, CheckConclusion, JobConfig } from "../types.ts"; -import { errorMessage, formatDuration, shortReason } from "../util.ts"; +import { + durationToMs, + errorMessage, + formatDuration, + shortReason, +} from "../util.ts"; import type { CacheBuildData, CacheBuildResult } from "./cacheBuild.ts"; import { ciRun } from "./metadata.ts"; import { ciStep, steps, traceName } from "./names.ts"; @@ -66,6 +71,8 @@ export const defineJob = ({ const config: JobConfig = typeof idOrConfig === "string" ? { id: idOrConfig } : idOrConfig; + checkMaxAge(config); + // Inside a run, curried job factories and matrices build a new job object // per call, so the same ID being registered again is expected. if (jobs.has(config.id) && !getRunScope()) { @@ -95,6 +102,29 @@ export const defineJob = ({ return job; }; +/** + * Fail a malformed or non-positive `cache.maxAge` when the job is defined. A + * zero age would make every snapshot too old, so concurrent builds would + * delete each other's. + */ +const checkMaxAge = (config: JobConfig): void => { + const maxAge = config.cache?.maxAge; + + if (maxAge === undefined) { + return; + } + + try { + if (durationToMs(maxAge) <= 0) { + throw new Error("It must be longer than zero."); + } + } catch (error) { + throw new CiUsageError( + `Job "${config.id}" has an invalid \`cache.maxAge\`. ${errorMessage(error)}`, + ); + } +}; + export const conclusionForError = (error: unknown): CheckConclusion => { return error instanceof CommandTimeoutError ? "timed_out" : "failure"; }; @@ -757,6 +787,7 @@ const snapshotBuilt = async ( const taken = await snapshotMachine(scope, { target, + cache: scope.config.cache, ...(run.build?.exclude ? { exclude: run.build.exclude } : {}), ...(run.build?.broken ? { broken: true } : {}), ...(run.build?.unnamed ? { unnamed: true } : {}), diff --git a/packages/ci/src/types.ts b/packages/ci/src/types.ts index a09e84dd0..5ca6fc7b3 100644 --- a/packages/ci/src/types.ts +++ b/packages/ci/src/types.ts @@ -187,6 +187,15 @@ export interface CacheConfig { * their own scope. */ scope?: "branch" | "global"; + + /** + * How old a cached snapshot may be before it is rebuilt, like `"1d"`. A + * snapshot older than this, counted from when it was taken, is a miss: the + * job builds again and the new snapshot takes the name. + * + * Without it a snapshot is reused until its key changes or it expires. + */ + maxAge?: Duration; } /** From d42ab5beb30877a9f24a9ccf0ca3dfdc8764fd6b Mon Sep 17 00:00:00 2001 From: Jack Williams Date: Tue, 13 Oct 2026 21:01:13 +0100 Subject: [PATCH 07/11] Update ci-trace-spans.md --- .changeset/ci-trace-spans.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/ci-trace-spans.md b/.changeset/ci-trace-spans.md index ac0d50c6a..3449b6911 100644 --- a/.changeset/ci-trace-spans.md +++ b/.changeset/ci-trace-spans.md @@ -2,6 +2,6 @@ "@inngest/ci": minor --- -Group a run's steps into trace spans (one per job, one per command, one per extra machine), name CI's own steps for what they do, and mark them as CI's work so a trace can tell them from yours. Give a job a `name` to change what its span is called. Every span has a kind (job, sandbox, command, snapshot, github or attempt), and each statement on a background process (`output`, `exited`, `kill`) is a row of its own. Saving a sandbox is one `snapshot` row. +Group a run's steps into trace spans (one per job, one per command, one per extra machine), name CI's own steps for what they do, and mark them as CI's work so a trace can tell them from yours. Saving a sandbox is one `snapshot` row. `@inngest/ci` now needs `inngest` 4.23.1 or later, which takes a snapshot's own trace span and always has the span API; older SDKs are no longer supported. From 477822cfe0f306aa99a52aae2a6bdf984d420aa1 Mon Sep 17 00:00:00 2001 From: x Date: Tue, 13 Oct 2026 20:08:08 +0000 Subject: [PATCH 08/11] Name a job's span from its config, keeping the number of later direct calls A later direct call has the path `id (n)`, which is not a registry key, so its span lost the configured name. The span now takes the name from the job's config and keeps the call number. Co-Authored-By: Claude Opus 5.5 --- packages/ci/src/machine/machine.ts | 2 +- packages/ci/src/pipeline/job.ts | 2 +- packages/ci/src/pipeline/names.ts | 20 +++++++++++++------- packages/ci/src/pipeline/scope.ts | 3 ++- packages/ci/src/pipeline/spans.test.ts | 16 +++++++++++++++- 5 files changed, 32 insertions(+), 11 deletions(-) diff --git a/packages/ci/src/machine/machine.ts b/packages/ci/src/machine/machine.ts index 9a36a46f1..1728667c4 100644 --- a/packages/ci/src/machine/machine.ts +++ b/packages/ci/src/machine/machine.ts @@ -43,7 +43,7 @@ import { export const ensureMachine = async ( scope: CiJobScope, ): Promise => { - scope.machine ??= inJobSpan(scope.run, scope.jobPath, () => { + scope.machine ??= inJobSpan(scope.run, scope.config, scope.jobPath, () => { return inMachineSpan(scope, () => { const stepId = joinId(scope.path, "machine"); diff --git a/packages/ci/src/pipeline/job.ts b/packages/ci/src/pipeline/job.ts index e482b6d8d..77d3a6f35 100644 --- a/packages/ci/src/pipeline/job.ts +++ b/packages/ci/src/pipeline/job.ts @@ -398,7 +398,7 @@ export const validateInput = async ( const jobBody = ( args: RunJobArgs & { run: CiRunScope; path: string; number: number }, ): Promise => { - return inJobSpan(args.run, args.path, () => { + return inJobSpan(args.run, args.config, args.path, () => { return jobSteps(args); }); }; diff --git a/packages/ci/src/pipeline/names.ts b/packages/ci/src/pipeline/names.ts index 475a259b5..f2ab09725 100644 --- a/packages/ci/src/pipeline/names.ts +++ b/packages/ci/src/pipeline/names.ts @@ -10,10 +10,9 @@ * @module */ -import type { CheckConclusion } from "../types.ts"; +import type { CheckConclusion, JobConfig } from "../types.ts"; import { formatDuration } from "../util.ts"; import { version } from "../version.ts"; -import type { CiRunScope } from "./scope.ts"; import { joinId, scopeSeparator } from "./scope.ts"; import type { SpanInfo, SpanKind } from "./spans.ts"; import { originOption } from "./spans.ts"; @@ -43,8 +42,8 @@ const ciSpan = (kind: SpanKind, id: string, name: string): SpanInfo => { */ export const spans = { /** A job, which holds everything it does. */ - job: (run: CiRunScope, path: string): SpanInfo => { - return userSpan("job", path, traceName.job(run, path)); + job: (config: JobConfig, path: string): SpanInfo => { + return userSpan("job", path, traceName.job(config, path)); }, /** The run's one span for reporting to GitHub. */ @@ -110,9 +109,16 @@ const outcomes: Partial> = { }; export const traceName = { - /** A job's span: its `name`, or its path when it has none. */ - job: (run: CiRunScope, path: string): string => { - return run.ci.jobs.get(path)?.config.name ?? path; + /** + * A job's span: its `name`, or its path when it has none. A later direct + * call, whose path is `${id} (n)`, keeps its `(n)` after the name. + */ + job: (config: JobConfig, path: string): string => { + if (!config.name) { + return path; + } + + return `${config.name}${path.slice(config.id.length)}`; }, /** The span a job's sandbox starts in. */ diff --git a/packages/ci/src/pipeline/scope.ts b/packages/ci/src/pipeline/scope.ts index b4f7932b4..d960c8340 100644 --- a/packages/ci/src/pipeline/scope.ts +++ b/packages/ci/src/pipeline/scope.ts @@ -554,11 +554,12 @@ const prefixStepId = (idOrOptions: unknown, prefix: string): unknown => { */ export const inJobSpan = ( run: CiRunScope, + config: JobConfig, jobPath: string, fn: () => R, ): R => { return runWithAsyncCtx(run.asyncCtx, () => { - return inSpan(spans.job(run, jobPath), fn); + return inSpan(spans.job(config, jobPath), fn); }); }; diff --git a/packages/ci/src/pipeline/spans.test.ts b/packages/ci/src/pipeline/spans.test.ts index 6e5f13b71..5dd3b06f6 100644 --- a/packages/ci/src/pipeline/spans.test.ts +++ b/packages/ci/src/pipeline/spans.test.ts @@ -8,7 +8,8 @@ import { describe, expect, test, vi } from "vitest"; import { snapshotMachine } from "../machine/machine.ts"; -import { ciOrigin } from "./names.ts"; +import type { JobConfig } from "../types.ts"; +import { ciOrigin, traceName } from "./names.ts"; import type { CiJobScope } from "./scope.ts"; import { originOption } from "./spans.ts"; @@ -55,3 +56,16 @@ describe("a snapshot's span", () => { ]); }); }); + +describe("a job's span name", () => { + const named = { id: "build", name: "Build it" } as JobConfig; + + test("is the job's name, and its id when it has none", () => { + expect(traceName.job(named, "build")).toBe("Build it"); + expect(traceName.job({ id: "build" } as JobConfig, "build")).toBe("build"); + }); + + test("keeps the name on a later direct call, with its number", () => { + expect(traceName.job(named, "build (2)")).toBe("Build it (2)"); + }); +}); From 5d039f37e4e26b27a4f6d80e021dd2ea6fba69e8 Mon Sep 17 00:00:00 2001 From: Jack Williams Date: Tue, 13 Oct 2026 20:29:04 +0000 Subject: [PATCH 09/11] Drop changeset: ci-shared-helpers is a pure internal refactor Co-Authored-By: Claude Opus 5.5 --- .changeset/ci-shared-helpers.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .changeset/ci-shared-helpers.md diff --git a/.changeset/ci-shared-helpers.md b/.changeset/ci-shared-helpers.md deleted file mode 100644 index 400014067..000000000 --- a/.changeset/ci-shared-helpers.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@inngest/ci": patch ---- - -Internal refactor: CI's step IDs and names come from one catalog, and snapshot naming, sandbox paging and snapshot deletes, cache lookups and check updates share helpers. Nothing user-visible changes. From f5a92c1a2c1d0554650adc79d9004968ae0f62ab Mon Sep 17 00:00:00 2001 From: Jack Williams Date: Tue, 13 Oct 2026 22:20:20 +0000 Subject: [PATCH 10/11] docs(ci): rewrite cache-max-age changeset as a release note Co-Authored-By: Claude Sonnet 5.5 --- .changeset/ci-cache-max-age.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.changeset/ci-cache-max-age.md b/.changeset/ci-cache-max-age.md index 3387a1b04..64bbeb788 100644 --- a/.changeset/ci-cache-max-age.md +++ b/.changeset/ci-cache-max-age.md @@ -2,4 +2,8 @@ "@inngest/ci": minor --- -Add `cache.maxAge`: a cached snapshot older than it counts as a miss, so the job builds again and the new snapshot takes the name. A malformed `maxAge` throws when the job is defined. +Cached jobs can now expire with `cache.maxAge`: a snapshot older than it is rebuilt. + +```ts +ci.job({ id: "base", cache: { key: files("images/node-base/**"), maxAge: "1d" } }, …); +``` From fac6789cc3fbadbaf5555c1b0808aada8f4f94b9 Mon Sep 17 00:00:00 2001 From: Jack Williams Date: Tue, 13 Oct 2026 23:42:07 +0000 Subject: [PATCH 11/11] docs(ci): finish the checkMaxAge comment Co-Authored-By: Claude Opus 5.5 --- packages/ci/src/pipeline/job.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ci/src/pipeline/job.ts b/packages/ci/src/pipeline/job.ts index c04b1a539..3b2c6809f 100644 --- a/packages/ci/src/pipeline/job.ts +++ b/packages/ci/src/pipeline/job.ts @@ -105,7 +105,7 @@ export const defineJob = ({ /** * Fail a malformed or non-positive `cache.maxAge` when the job is defined. A * zero age would make every snapshot too old, so concurrent builds would - * delete each other's. + * delete each other's snapshots. */ const checkMaxAge = (config: JobConfig): void => { const maxAge = config.cache?.maxAge;