From e5bf55339f8aaf05dd294228bfc50fab8363ab70 Mon Sep 17 00:00:00 2001 From: x Date: Fri, 9 Oct 2026 23:02:42 +0000 Subject: [PATCH 01/18] feat(ci): group a run's steps into trace spans and name CI's own steps Jobs, commands and extra machines become spans, CI's steps are named for what they do and marked as CI's work. A job's `name` sets its span name. Co-Authored-By: Claude Opus 5.5 --- .changeset/ci-trace-spans.md | 5 + packages/ci/src/cache/cache.ts | 28 +- packages/ci/src/checkout/changed.ts | 3 +- packages/ci/src/checkout/wait.ts | 4 +- packages/ci/src/github/checks.test.ts | 3 +- packages/ci/src/github/checks.ts | 161 +++-- packages/ci/src/github/github.test.ts | 2 +- packages/ci/src/github/helpers.ts | 114 ++-- packages/ci/src/machine/command.ts | 172 ++++-- packages/ci/src/machine/from.ts | 15 +- packages/ci/src/machine/machine.ts | 150 +++-- packages/ci/src/machine/spans.test.ts | 583 ++++++++++++++++++ packages/ci/src/pipeline/README.md | 2 + .../ci/src/pipeline/cacheGuarantees.test.ts | 12 + packages/ci/src/pipeline/durable.ts | 3 +- packages/ci/src/pipeline/job.ts | 46 +- packages/ci/src/pipeline/metadata.test.ts | 9 + packages/ci/src/pipeline/metadata.ts | 3 +- packages/ci/src/pipeline/names.ts | 160 +++++ packages/ci/src/pipeline/pipeline.ts | 102 +-- packages/ci/src/pipeline/scope.ts | 59 +- packages/ci/src/pipeline/spans.test.ts | 146 +++++ packages/ci/src/pipeline/spans.ts | 70 +++ packages/ci/src/report.ts | 5 +- packages/ci/src/testing/README.md | 1 + packages/ci/src/testing/runFunction.ts | 43 +- packages/ci/src/testing/setup.ts | 11 + packages/ci/src/testing/spanStub.ts | 185 ++++++ packages/ci/src/types.ts | 4 +- packages/ci/src/units.test.ts | 8 + packages/ci/vitest.config.ts | 1 + 31 files changed, 1825 insertions(+), 285 deletions(-) create mode 100644 .changeset/ci-trace-spans.md create mode 100644 packages/ci/src/machine/spans.test.ts create mode 100644 packages/ci/src/pipeline/names.ts create mode 100644 packages/ci/src/pipeline/spans.test.ts create mode 100644 packages/ci/src/pipeline/spans.ts create mode 100644 packages/ci/src/testing/setup.ts create mode 100644 packages/ci/src/testing/spanStub.ts diff --git a/.changeset/ci-trace-spans.md b/.changeset/ci-trace-spans.md new file mode 100644 index 000000000..d3ef514f5 --- /dev/null +++ b/.changeset/ci-trace-spans.md @@ -0,0 +1,5 @@ +--- +"@inngest/ci": minor +--- + +Group a run's steps into trace spans (one per job, one per command, one per extra machine), name CI's own steps for what they do, and mark them as CI's work so a trace can tell them from yours. Give a job a `name` to change what its span is called. diff --git a/packages/ci/src/cache/cache.ts b/packages/ci/src/cache/cache.ts index 2d5d99fb2..d53d856b0 100644 --- a/packages/ci/src/cache/cache.ts +++ b/packages/ci/src/cache/cache.ts @@ -11,6 +11,7 @@ */ import { tagStep } from "../pipeline/metadata.ts"; +import { ciStep, traceName } from "../pipeline/names.ts"; import type { CiJobScope, CiRunScope } from "../pipeline/scope.ts"; import { countApi, rootRunIdOf, scopeSeparator } from "../pipeline/scope.ts"; import type { @@ -255,7 +256,7 @@ export interface CachedSnapshot { export interface CacheTarget { /** The job's resolved key. */ ownKey: string; - /** The name the snapshot is given, in the scope the job writes. */ + /** The name a build gives the snapshot, in the scope the job writes. */ name: string; } @@ -282,10 +283,7 @@ export const cacheTarget = async ( countApi("cache"); const ownKey = (await run.step.run( - { - id: `${job.path}${scopeSeparator}cache:key`, - name: "cache:key", - }, + ciStep(`${job.path}${scopeSeparator}cache:key`, traceName.checkCache), async () => { await tagStep(run, { kind: "cache", job: job.path }); @@ -337,7 +335,7 @@ const isExpiring = (expiresAt: string | undefined): boolean => { return Number.isFinite(at) && at - Date.now() < expiryMarginMs; }; -/** How long a lookup waits for a snapshot another run is still taking. */ +/** How long a lookup waits for a snapshot another build is still taking. */ const readyWaitMs = 2 * 60 * 1000; const readyPollMs = 1000; @@ -387,7 +385,7 @@ const waitUntilReady = async ( /** * The newest snapshot with exactly this name, if it can be used: ready, or - * ready after waiting for the run taking it, and not about to expire. + * ready after waiting for the build taking it, and not about to expire. * * Any error is a miss, and only an exact name counts, since a server that * doesn't know names ignores the filter and lists every snapshot. @@ -464,10 +462,7 @@ export const lookupCache = async ( const { run } = scope; const found = (await run.step.run( - { - id: `${scope.path}${scopeSeparator}cache:lookup`, - name: "cache:lookup", - }, + ciStep(`${scope.path}${scopeSeparator}cache:lookup`, traceName.lookUpCache), async () => { await tagStep(run, { kind: "cache", job: scope.path }); @@ -521,10 +516,7 @@ export const lookupBeforeBuild = async ( exclude?: string, ): Promise => { const found = (await run.step.run( - { - id: `${job.stepPath}${scopeSeparator}lookup`, - name: "cache:lookup", - }, + ciStep(`${job.stepPath}${scopeSeparator}lookup`, traceName.lookUpCache), async () => { await tagStep(run, { kind: "cache", job: job.path }); @@ -555,7 +547,7 @@ export const resolveTakenName = async ( broken?: boolean, ): Promise<{ winner?: CachedSnapshot; cleared: boolean }> => { return (await run.step.run( - { id: stepId, name: "cache:name-taken" }, + ciStep(stepId, traceName.resolveCacheName), async () => { const winner = await findNamed(run, name, exclude); @@ -605,7 +597,7 @@ export const snapshotState = async ( snapshotId: string, ): Promise<"gone" | "creating" | "ready"> => { return (await run.step.run( - { id: stepId, name: "cache:snapshot-state" }, + ciStep(stepId, traceName.checkSnapshotState), async () => { try { const snapshot = (await snapshotsClient(run).get(snapshotId)) as @@ -637,7 +629,7 @@ export const deleteSnapshot = async ( ): Promise => { try { const result = (await run.step.run( - { id: stepId, name: "cache:delete" }, + ciStep(stepId, traceName.deleteBadSnapshot), async () => { try { const snapshot = await snapshotsClient(run).get(snapshotId); diff --git a/packages/ci/src/checkout/changed.ts b/packages/ci/src/checkout/changed.ts index 3e8433093..8fad40dce 100644 --- a/packages/ci/src/checkout/changed.ts +++ b/packages/ci/src/checkout/changed.ts @@ -6,6 +6,7 @@ */ import { CiUsageError } from "../errors.ts"; +import { traceName } from "../pipeline/names.ts"; import type { CiRunScope } from "../pipeline/scope.ts"; import { countApi, getRunScope } from "../pipeline/scope.ts"; import type { RepoContext } from "../types.ts"; @@ -91,7 +92,7 @@ export const changedFiles = async (): Promise => { const readChangedFiles = async (run: CiRunScope): Promise => { const files = (await run.step.run( - { id: "changed", name: "changed" }, + { id: "changed", name: traceName.findChangedFiles }, async () => { try { return await listChangedFiles(run.repo); diff --git a/packages/ci/src/checkout/wait.ts b/packages/ci/src/checkout/wait.ts index b91f9f762..526794143 100644 --- a/packages/ci/src/checkout/wait.ts +++ b/packages/ci/src/checkout/wait.ts @@ -81,7 +81,7 @@ export const waitForHttp = async ( return target; }, ["/bin/sh", "-c", script], - `waitForHttp ${url}`, + { label: `waitForHttp ${url}` }, ).timeout(`${timeoutMs + waitHeadroomMs}ms`); }; @@ -125,6 +125,6 @@ export const waitForPort = async ( return target; }, ["/bin/sh", "-c", script], - `waitForPort ${port}`, + { label: `waitForPort ${port}` }, ).timeout(`${timeoutMs + waitHeadroomMs}ms`); }; diff --git a/packages/ci/src/github/checks.test.ts b/packages/ci/src/github/checks.test.ts index ff38f43a2..4a6ae6077 100644 --- a/packages/ci/src/github/checks.test.ts +++ b/packages/ci/src/github/checks.test.ts @@ -42,12 +42,11 @@ const fakeRun = (overrides: Partial = {}): CiRunScope => { jobCalls: new Map(), machines: new Map(), snapshots: new Map(), + cached: new Map(), createdSnapshots: new Set(), - cachedSnapshots: new Map(), sandboxes: new Set(), summaries: [], counters: new Map(), - snapshotsUnavailable: false, warnings: [], pipelineSummaries: [], pipelineAnnotations: [], diff --git a/packages/ci/src/github/checks.ts b/packages/ci/src/github/checks.ts index 8f4fa5232..f6a36c54e 100644 --- a/packages/ci/src/github/checks.ts +++ b/packages/ci/src/github/checks.ts @@ -6,7 +6,9 @@ */ import { type StepTag, tagStep } from "../pipeline/metadata.ts"; +import { traceName } from "../pipeline/names.ts"; import type { CiRunScope } from "../pipeline/scope.ts"; +import { inGitHubSpan } from "../pipeline/scope.ts"; import type { CheckAnnotation, CheckConclusion } from "../types.ts"; import { errorMessage, formatDuration } from "../util.ts"; import type { GitHubProvider, Octokit } from "./auth.ts"; @@ -225,6 +227,17 @@ export const normaliseAnnotation = ( }; }; +/** Run a check update as a step in the run's GitHub span. */ +const githubStep = ( + run: CiRunScope, + step: { id: string; name: string }, + fn: () => Promise, +): Promise => { + return inGitHubSpan(run, () => { + return run.step.run(step, fn) as Promise; + }); +}; + /** * Build the reporter used by every pipeline run. * @@ -262,20 +275,17 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { run: CiRunScope, key: string, name: string, - stepId: string, + step: { id: string; name: string }, tag: StepTag, metadata?: () => Record, ) => { - const result = await run.step.run( - { id: stepId, name: stepId }, - async () => { - await tagStep(run, tag, metadata?.()); + const result = await githubStep(run, step, async () => { + await tagStep(run, tag, metadata?.()); - const started = await sink.start({ run, name, ...identity(run, key) }); + const started = await sink.start({ run, name, ...identity(run, key) }); - return { ...started, startedAt: Date.now() }; - }, - ); + return { ...started, startedAt: Date.now() }; + }); if (result?.id) { checkRunIds.set(idKey(run, key), result.id); @@ -288,32 +298,29 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { run: CiRunScope, key: string, name: string, - stepId: string, + step: { id: string; name: string }, result: () => CheckResult, tag: StepTag, metadata?: () => Record, ): Promise => { - const endedAt = await run.step.run( - { id: stepId, name: stepId }, - async () => { - await tagStep(run, tag, metadata?.()); + const endedAt = await githubStep(run, step, async () => { + await tagStep(run, tag, metadata?.()); - const read = result(); + const read = result(); - await sink.complete({ - run, - name, - ...identity(run, key), - conclusion: read.conclusion, - title: read.title, - summary: truncateSummary(read.summary ?? ""), - annotations: (read.annotations ?? []).map(normaliseAnnotation), - ...idFor(run, key), - }); + await sink.complete({ + run, + name, + ...identity(run, key), + conclusion: read.conclusion, + title: read.title, + summary: truncateSummary(read.summary ?? ""), + annotations: (read.annotations ?? []).map(normaliseAnnotation), + ...idFor(run, key), + }); - return Date.now(); - }, - ); + return Date.now(); + }); checkRunIds.delete(idKey(run, key)); @@ -330,7 +337,10 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { run, "pipeline", run.checkName, - `github › check:${run.checkName}:start`, + { + id: `github › check:${run.checkName}:start`, + name: traceName.createCheck(run.checkName), + }, { kind: "check" }, metadata, ); @@ -345,7 +355,10 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { run, "pipeline", run.checkName, - `github › check:${run.checkName}:complete`, + { + id: `github › check:${run.checkName}:complete`, + name: traceName.completeCheck(run.checkName), + }, result, { kind: "check" }, metadata, @@ -361,7 +374,10 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { run, jobPath, jobCheckName(run, jobPath, name), - `github › check:${jobPath}:start`, + { + id: `github › check:${jobPath}:start`, + name: traceName.report(jobPath, "started"), + }, { kind: "check", job: jobPath }, ); }, @@ -375,7 +391,10 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { run, jobPath, jobCheckName(run, jobPath, name), - `github › check:${jobPath}:complete`, + { + id: `github › check:${jobPath}:complete`, + name: traceName.report(jobPath, traceName.outcome(result.conclusion)), + }, () => { return result; }, @@ -394,31 +413,31 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return closing(read()); } - const stepId = "github › check:jobs:complete"; + const step = { + id: "github › check:jobs:complete", + name: traceName.report("jobs", "ended with the run"), + }; - const closed = (await run.step.run( - { id: stepId, name: stepId }, - async () => { - await tagStep(run, { kind: "check" }); - - const jobs = read(); - - for (const { jobPath, name, ...result } of jobs) { - await sink.complete({ - run, - name: jobCheckName(run, jobPath, name), - ...identity(run, jobPath), - conclusion: result.conclusion, - title: result.title, - summary: truncateSummary(result.summary ?? ""), - annotations: (result.annotations ?? []).map(normaliseAnnotation), - ...idFor(run, jobPath), - }); - } + const closed = await githubStep(run, step, async () => { + await tagStep(run, { kind: "check" }); - return closing(jobs); - }, - )) as Array>; + const jobs = read(); + + for (const { jobPath, name, ...result } of jobs) { + await sink.complete({ + run, + name: jobCheckName(run, jobPath, name), + ...identity(run, jobPath), + conclusion: result.conclusion, + title: result.title, + summary: truncateSummary(result.summary ?? ""), + annotations: (result.annotations ?? []).map(normaliseAnnotation), + ...idFor(run, jobPath), + }); + } + + return closing(jobs); + }); // Outside the step, so a replay also forgets the IDs its memoized start // steps put back. @@ -439,9 +458,15 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { jobPath === undefined ? run.checkName : jobCheckName(run, jobPath, name); - const stepId = `github › check:${key}:retry:${run.attempt}`; + const step = { + id: `github › check:${key}:retry:${run.attempt}`, + name: traceName.report( + jobPath ?? run.checkName, + traceName.retrying(run.attempt + 2, run.maxAttempts), + ), + }; - await run.step.run({ id: stepId, name: stepId }, async () => { + await githubStep(run, step, async () => { await tagStep( run, jobPath === undefined @@ -465,9 +490,15 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return; } - const stepId = `github › check:jobs:retry:${run.attempt}`; + const step = { + id: `github › check:jobs:retry:${run.attempt}`, + name: traceName.report( + "jobs", + traceName.retrying(run.attempt + 2, run.maxAttempts), + ), + }; - await run.step.run({ id: stepId, name: stepId }, async () => { + await githubStep(run, step, async () => { await tagStep(run, { kind: "check" }); for (const { jobPath, name } of read()) { @@ -490,10 +521,11 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { const message = errorMessage(error).split("\n")[0]; - await run.step.run( + await githubStep( + run, { id: `github › check:${jobPath}:attempt:${attempt}`, - name: `check:${jobPath}:attempt:${attempt}`, + name: traceName.report(jobPath, traceName.retrying(attempt + 1, of)), }, async () => { await sink.update?.({ @@ -527,9 +559,12 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { } const { parent } = run.build; - const stepId = `github › check:${parent.jobPath}:building`; + const step = { + id: `github › check:${parent.jobPath}:building`, + name: traceName.report(parent.jobPath, "building"), + }; - await run.step.run({ id: stepId, name: stepId }, async () => { + await githubStep(run, step, async () => { await sink.update?.({ // The check belongs to the pipeline that invoked this run. run: { ...run, pipelineId: parent.pipelineId }, diff --git a/packages/ci/src/github/github.test.ts b/packages/ci/src/github/github.test.ts index 35dca03b7..800265fd1 100644 --- a/packages/ci/src/github/github.test.ts +++ b/packages/ci/src/github/github.test.ts @@ -455,7 +455,7 @@ describe("github helpers", () => { { event: prEvent, resolveWait: (step) => { - return step.displayName?.includes("waitForChecks") + return step.displayName === "Wait for check: vercel" ? { name: "github/check_run.completed", data: { check_run: { name: "vercel", conclusion: "failure" } }, diff --git a/packages/ci/src/github/helpers.ts b/packages/ci/src/github/helpers.ts index 17867638e..1e6abc3e9 100644 --- a/packages/ci/src/github/helpers.ts +++ b/packages/ci/src/github/helpers.ts @@ -8,10 +8,13 @@ import { RetryAfterError } from "inngest"; import { CiUsageError } from "../errors.ts"; import { durablePath } from "../pipeline/durable.ts"; +import { traceName } from "../pipeline/names.ts"; +import type { CiRunScope } from "../pipeline/scope.ts"; import { countApi, getJobScope, getRunScope, + inGitHubSpan, requireRunScope, } from "../pipeline/scope.ts"; import type { CheckConclusion, Duration } from "../types.ts"; @@ -20,6 +23,14 @@ import type { Octokit } from "./auth.ts"; import { mapGitHubError, octokitForRun, rest } from "./rest.ts"; import { hasPermission, type Permission } from "./triggers.ts"; +/** + * Run a helper's steps in the job that called it, or in the run's GitHub span + * when no job did. + */ +const inHelperSpan = (run: CiRunScope, fn: () => T): T => { + return getJobScope() ? fn() : inGitHubSpan(run, fn); +}; + /** * Run a helper's body as a single step, so all the calls it makes retry * together. Inside the step, `github.rest` calls run directly. @@ -28,6 +39,8 @@ const helperStep = async ( helper: string, key: string, fn: () => Promise, + /** The step's name, which is the helper's method name by default. */ + name = traceName.githubHelper(helper), ): Promise => { const run = requireRunScope(`github.${helper}`); @@ -40,7 +53,9 @@ const helperStep = async ( const job = getJobScope(); const id = `${job ? `${job.path} › ` : ""}github › ${helper}:${key}`; - return run.step.run({ id, name: id }, fn) as Promise; + return inHelperSpan(run, () => { + return run.step.run({ id, name }, fn) as Promise; + }); }; /** Run `call` as a step when inside a run, or directly when outside one. */ @@ -458,30 +473,35 @@ export const canUser = async ( login: string, permission: Permission, ): Promise => { - return helperStep("canUser", `${login}:${permission}`, async () => { - try { - const result = await rest.repos.getCollaboratorPermissionLevel({ - username: login, - }); + return helperStep( + "canUser", + `${login}:${permission}`, + async () => { + try { + const result = await rest.repos.getCollaboratorPermissionLevel({ + username: login, + }); - return hasPermission(result.permission, permission); - } catch (error) { - const status = - (error as ErrorWithStatus).status ?? - (error as ErrorWithStatus).cause?.status; - - // A 403 or 404 here means "can't see it", which is the same as "no". - // Rate limits and server errors must retry, not become a "no". - if ( - (status === 403 || status === 404) && - !(error instanceof RetryAfterError) - ) { - return false; - } + return hasPermission(result.permission, permission); + } catch (error) { + const status = + (error as ErrorWithStatus).status ?? + (error as ErrorWithStatus).cause?.status; - throw error; - } - }); + // A 403 or 404 here means "can't see it", which is the same as "no". + // Rate limits and server errors must retry, not become a "no". + if ( + (status === 403 || status === 404) && + !(error instanceof RetryAfterError) + ) { + return false; + } + + throw error; + } + }, + traceName.canUser(login, permission), + ); }; /** @@ -537,18 +557,20 @@ export const waitForChecks = async (opts: { const waited = await Promise.all( missing.map(async (name) => { - const event = (await run.step.waitForEvent( - { - id: `github › waitForChecks:${name}`, - name: `waitForChecks:${name}`, - }, - { - event: "github/check_run.completed", - timeout: opts.timeout ?? "1h", - if: `async.data.check_run.name == ${JSON.stringify(name)} && async.data.check_run.head_sha == ${JSON.stringify(sha)}`, - }, + const event = (await inHelperSpan(run, () => { + return run.step.waitForEvent( + { + id: `github › waitForChecks:${name}`, + name: traceName.waitForCheck(name), + }, + { + event: "github/check_run.completed", + timeout: opts.timeout ?? "1h", + if: `async.data.check_run.name == ${JSON.stringify(name)} && async.data.check_run.head_sha == ${JSON.stringify(sha)}`, + }, + ); // biome-ignore lint/suspicious/noExplicitAny: event shape is the user's - )) as any; + })) as any; return [ name, @@ -597,18 +619,20 @@ export const waitForWorkflow = async (opts: { ? `async.data.workflow_run.workflow_id == ${opts.workflow}` : `async.data.workflow_run.path.endsWith(${JSON.stringify(`/${opts.workflow}`)})`; - const event = (await run.step.waitForEvent( - { - id: `github › waitForWorkflow:${opts.workflow}`, - name: `waitForWorkflow:${opts.workflow}`, - }, - { - event: "github/workflow_run.completed", - timeout: opts.timeout ?? "1h", - if: `async.data.workflow_run.head_sha == ${JSON.stringify(sha)} && ${workflowMatch}`, - }, + const event = (await inHelperSpan(run, () => { + return run.step.waitForEvent( + { + id: `github › waitForWorkflow:${opts.workflow}`, + name: traceName.waitForWorkflow(opts.workflow), + }, + { + event: "github/workflow_run.completed", + timeout: opts.timeout ?? "1h", + if: `async.data.workflow_run.head_sha == ${JSON.stringify(sha)} && ${workflowMatch}`, + }, + ); // biome-ignore lint/suspicious/noExplicitAny: event shape is the user's - )) as any; + })) as any; return (event?.data?.workflow_run?.conclusion ?? "timed_out") as | CheckConclusion diff --git a/packages/ci/src/machine/command.ts b/packages/ci/src/machine/command.ts index 46459e8d2..6e0a0f732 100644 --- a/packages/ci/src/machine/command.ts +++ b/packages/ci/src/machine/command.ts @@ -6,12 +6,14 @@ * @module */ +import { NonRetriableError, StepError } from "inngest"; import { getSandboxError } from "inngest/experimental"; import { CiUsageError, CommandFailedError, CommandTimeoutError, } from "../errors.ts"; +import { ciSpan, ciStep, traceName } from "../pipeline/names.ts"; import type { CiJobScope, MachineHandle } from "../pipeline/scope.ts"; import { countApi, @@ -20,6 +22,7 @@ import { requireJobScope, scopeSeparator, } from "../pipeline/scope.ts"; +import { inSpan } from "../pipeline/spans.ts"; import type { BackgroundProcess, Command, @@ -35,7 +38,7 @@ import { tail, truncateLabel, } from "../util.ts"; -import { ensureMachine } from "./machine.ts"; +import { ensureMachine, inMachineSpan } from "./machine.ts"; /** * Captured `commands.run` is capped at five minutes, so anything longer runs @@ -61,6 +64,9 @@ type SandboxProcess = any; interface CommandState { argv: string[]; + /** What the command reads as, when that isn't its argv, as for `$.sh`. */ + text?: string; + /** A name for the command, from `.as()` or the helper that made it. */ label?: string; env: Record; cwd?: string; @@ -165,11 +171,12 @@ export const buildShellString = ( return out; }; -/** The id and name of a step nested under `stepId`. */ -const subStep = (stepId: string, suffix: string) => { - const id = `${stepId}${scopeSeparator}${suffix}`; - - return { id, name: id }; +/** + * A step nested under `stepId`, named for what it does. It's CI's work: the + * command's span is yours, but how CI runs it isn't. + */ +const subStep = (stepId: string, suffix: string, name: string) => { + return ciStep(`${stepId}${scopeSeparator}${suffix}`, name); }; class CommandBuilder implements Command { @@ -261,11 +268,10 @@ class CommandBuilder implements Command { const stepId = this.stepId(scope); const machine = await ensureMachine(scope); - const process = await startProcess( - machine, - stepId, - this.spawnOptions(scope), - ); + // Each call back into the process re-enters the command's span. + const process = await this.inSpan(scope, stepId, () => { + return startProcess(machine, stepId, this.spawnOptions(scope)); + }); const argv = this.state.argv; const secrets = this.secretValues(scope); @@ -275,25 +281,36 @@ class CommandBuilder implements Command { return { id: process.id, - exited: async () => { - const polled = await pollUntilTerminal({ - scope, - machine, - process, - stepId, - nextWait, + exited: () => { + return this.inSpan(scope, stepId, async () => { + const polled = await pollUntilTerminal({ + scope, + machine, + process, + stepId, + nextWait, + }); + + return readResult({ process: polled.process, stepId, argv, secrets }); }); - - return readResult({ process: polled.process, stepId, argv, secrets }); }, kill: async (signal = 15) => { - await process.signal(subStep(stepId, "kill"), { signal }); + await this.inSpan(scope, stepId, () => { + return process.signal( + subStep(stepId, "kill", traceName.stopProcess), + { signal }, + ); + }); }, output: async (opts) => { - const output = await process.getOutput( - subStep(stepId, `output #${nextWait()}`), - { tailBytes: opts?.tailBytes ?? outputTailBytes }, - ); + const output = await this.inSpan(scope, stepId, () => { + return process.getOutput( + subStep(stepId, `output #${nextWait()}`, traceName.readOutput), + { + tailBytes: opts?.tailBytes ?? outputTailBytes, + }, + ); + }); const decoded = decodeChunks(output); @@ -313,8 +330,14 @@ class CommandBuilder implements Command { return this.started; } + /** What the command reads as: its `$.sh` script, or its arguments. */ + private commandText(): string { + return this.state.text ?? truncateLabel(this.state.argv.join(" ")); + } + + /** What the command's step IDs are built from. */ private labelText(): string { - return this.state.label ?? truncateLabel(this.state.argv.join(" ")); + return this.state.label ?? this.commandText(); } private stepId(scope: CiJobScope): string { @@ -348,6 +371,22 @@ class CommandBuilder implements Command { }); } + /** Run `fn` in the command's span, in its machine's span if it's an extra. */ + private inSpan(scope: CiJobScope, stepId: string, fn: () => T): T { + const span = { + id: stepId, + name: traceName.command(this.commandText(), this.state.label), + }; + + return inMachineSpan(scope, () => { + return inSpan(span, fn); + }); + } + + /** + * Each attempt runs in the command's span, and in a span of its own when the + * command has retries. A failed attempt's span ends in its failure. + */ private async runWithRetries(): Promise { const scope = this.getScope(); const stepId = this.stepId(scope); @@ -361,7 +400,24 @@ class CommandBuilder implements Command { scope.run.ci.hooks.commandStarted(scope, attemptInfo); - const result = await this.runOnce(scope, attemptId); + const runAttempt = async () => { + const result = await this.runOnce(scope, attemptId); + + if (result.exitCode !== 0 && !this.state.nothrow) { + await recordFailure(scope, attemptId, result.exitCode); + } + + return result; + }; + + const result = await this.inSpan(scope, stepId, () => { + return attempts === 1 + ? runAttempt() + : inSpan( + ciSpan(`attempt-${attempt}`, traceName.attempt(attempt)), + runAttempt, + ); + }); scope.run.ci.hooks.commandFinished(scope, attemptInfo, result); @@ -422,7 +478,7 @@ class CommandBuilder implements Command { try { const result = await machine.sandbox.commands.run( - { id: stepId, name: stepId }, + ciStep(stepId, traceName.runAndReadOutput), command, { ...options, timeout: timeoutMs }, ); @@ -473,7 +529,10 @@ class CommandBuilder implements Command { if (polled.timedOut) { await this.state.onTimeout?.(); - await process.signal(subStep(stepId, "timeout-kill"), { signal: 9 }); + await process.signal( + subStep(stepId, "timeout-kill", traceName.stopAfterTimeout), + { signal: 9 }, + ); throw this.timeoutError(scope); } @@ -500,6 +559,31 @@ const counter = (): (() => number) => { }; }; +/** + * Record that a run of a command failed, as a step that fails without retrying. + * The sandbox steps before it all succeed, since a non-zero exit is a result, + * so this step is what shows the run as failed in the trace. Its error is + * caught: the caller decides whether to retry or throw. + */ +const recordFailure = async ( + scope: CiJobScope, + stepId: string, + exitCode: number, +): Promise => { + try { + await scope.run.step.run( + subStep(stepId, "exit", traceName.exited(exitCode)), + () => { + throw new NonRetriableError(`exit ${exitCode}`); + }, + ); + } catch (error) { + if (!(error instanceof StepError)) { + throw error; + } + } +}; + /** * Start a managed process. Cloud can answer a start that succeeded with an * ambiguous error, in which case the process that did start is adopted @@ -520,7 +604,7 @@ const startProcess = async ( try { const process = await machine.sandbox.processes.start( - subStep(stepId, "start"), + subStep(stepId, "start", traceName.startProcess), options, ); @@ -570,7 +654,7 @@ const adoptAmbiguousStart = async ( } const listed = (await machine.sandbox.processes.list( - subStep(stepId, "reconcile"), + subStep(stepId, "reconcile", traceName.findStartedProcess), { limit: 250 }, )) as { items: Array<{ @@ -634,7 +718,7 @@ const pollUntilTerminal = async (opts: { const interval = pollIntervalsMs[index - 1] ?? maxPollIntervalMs; await opts.scope.run.step.sleep( - subStep(opts.stepId, `wait #${index}`), + subStep(opts.stepId, `wait #${index}`, traceName.wait(interval)), interval, ); @@ -642,7 +726,7 @@ const pollUntilTerminal = async (opts: { current = (await opts.machine.sandbox.processes.get( - subStep(opts.stepId, `check #${index}`), + subStep(opts.stepId, `check #${index}`, traceName.pollProcess), current.id, )) ?? current; } @@ -656,9 +740,12 @@ const readResult = async (opts: { argv: string[]; secrets: string[]; }): Promise => { - const output = await opts.process.getOutput(subStep(opts.stepId, "output"), { - tailBytes: outputTailBytes, - }); + const output = await opts.process.getOutput( + subStep(opts.stepId, "output", traceName.readOutput), + { + tailBytes: outputTailBytes, + }, + ); const decoded = decodeChunks(output); const stdout = tail(decoded.stdout, outputTailBytes); @@ -756,11 +843,12 @@ const decodeChunks = (output: { export const createRawCommand = ( getScope: () => CiJobScope, argv: string[], - label?: string, + /** How the command reads, when not as its arguments. */ + display: Pick = {}, ): Command => { return new CommandBuilder(getScope, { argv, - ...(label === undefined ? {} : { label }), + ...display, env: {}, retries: 0, nothrow: false, @@ -780,11 +868,9 @@ export const createCommandTag = ( tag.sh = (strings: TemplateStringsArray, ...values: CommandValue[]) => { const rendered = buildShellString([...strings], values); - return createRawCommand( - getScope, - ["/bin/sh", "-c", rendered], - truncateLabel(rendered), - ); + return createRawCommand(getScope, ["/bin/sh", "-c", rendered], { + text: truncateLabel(rendered), + }); }; return tag; diff --git a/packages/ci/src/machine/from.ts b/packages/ci/src/machine/from.ts index c2aa06dc4..7c1a7ed03 100644 --- a/packages/ci/src/machine/from.ts +++ b/packages/ci/src/machine/from.ts @@ -23,7 +23,12 @@ import { CiUsageError } from "../errors.ts"; import type { CacheBuildResult } from "../pipeline/cacheBuild.ts"; import { adoptBuilt, invokeBuild, validateInput } from "../pipeline/job.ts"; import type { CiJobScope, CiRunScope } from "../pipeline/scope.ts"; -import { countApi, rebuildSuffix, scopeSeparator } from "../pipeline/scope.ts"; +import { + countApi, + outsideJobs, + rebuildSuffix, + scopeSeparator, +} from "../pipeline/scope.ts"; import type { AnyJob, JobConfig, JobRef } from "../types.ts"; import { errorMessage, hash, stableStringify } from "../util.ts"; @@ -329,7 +334,7 @@ const buildOf = ( return existing; } - const built = (async () => { + const built = outsideJobs(run, async () => { const base = await baseOf(run, parent); const identity = base @@ -359,7 +364,7 @@ const buildOf = ( adoptBuilt(run, result); return result; - })(); + }); run.builds.set(path, built); @@ -472,7 +477,7 @@ const requestRebuild = ({ return existing; } - const built = (async () => { + const built = outsideJobs(run, async () => { let unnamed = replacing.unnamed; // Here, not in each child, so a snapshot that every child found broken is @@ -513,7 +518,7 @@ const requestRebuild = ({ adoptBuilt(run, result); return result; - })(); + }); run.builds.set(path, built); diff --git a/packages/ci/src/machine/machine.ts b/packages/ci/src/machine/machine.ts index ea328421e..a0c4f6779 100644 --- a/packages/ci/src/machine/machine.ts +++ b/packages/ci/src/machine/machine.ts @@ -9,6 +9,7 @@ import type { Inngest } from "inngest"; import type { CachedSnapshot, CacheTarget } from "../cache/cache.ts"; import { resolveTakenName, snapshotState } from "../cache/cache.ts"; import { CiUsageError } from "../errors.ts"; +import { ciSpan, ciStep, traceName } from "../pipeline/names.ts"; import type { CiJobScope, CiRunScope, @@ -16,10 +17,12 @@ import type { } from "../pipeline/scope.ts"; import { defaultCwd, + inJobSpan, isRestoring, runInScope, scopeSeparator, } from "../pipeline/scope.ts"; +import { inSpan } from "../pipeline/spans.ts"; import type { MachineConfig } from "../types.ts"; import { boundedName, @@ -54,12 +57,25 @@ export const machineName = (runId: string, path: string): string => { * Create this scope's machine if it doesn't have one yet. * * Machines are lazy: a job that never runs a command never gets one, and - * concurrent first commands share a single creation promise. + * concurrent first commands share a single creation promise. Its steps are in + * a span of its own, in its job's span rather than the first command's. */ export const ensureMachine = async ( scope: CiJobScope, ): Promise => { - scope.machine ??= createMachine(scope); + scope.machine ??= inJobSpan(scope.run, scope.jobPath, () => { + return inMachineSpan(scope, () => { + const stepId = `${scope.path}${scopeSeparator}machine`; + + // Everything in it is CI's work, including falling back from a + // snapshot that won't start. + const span = ciSpan(stepId, traceName.startMachine(scope.fromJobIds[0])); + + return inSpan(span, () => { + return createMachine(scope, stepId); + }); + }); + }); const handle = await scope.machine; @@ -84,6 +100,23 @@ export const ensureMachine = async ( return handle; }; +/** + * Run `fn` in an extra machine's span, so its commands sit apart from the + * job's own. A job's own machine has none: the job's span is its span. + */ +export const inMachineSpan = (scope: CiJobScope, fn: () => R): R => { + if (scope.path === scope.jobPath) { + return fn(); + } + + const span = { + id: scope.path, + name: traceName.extraMachine(scope.path, scope.jobPath), + }; + + return inSpan(span, fn); +}; + /** * Run once on every machine before its first command. * @@ -104,7 +137,11 @@ interface Started { sandbox: any; } -const createMachine = async (scope: CiJobScope): Promise => { +const createMachine = async ( + scope: CiJobScope, + /** The create step's ID, which other steps for the machine build on. */ + stepId: string, +): Promise => { const { run } = scope; const tools = run.sandboxTools; @@ -115,7 +152,7 @@ const createMachine = async (scope: CiJobScope): Promise => { } const name = machineName(run.runId, scope.path); - const stepId = `${scope.path}${scopeSeparator}machine`; + const create = { id: stepId, name: traceName.createMachine }; const machineConfig = resolveMachineConfig( scope.config.machine ?? run.machine ?? run.ci.defaultMachine, @@ -123,27 +160,29 @@ const createMachine = async (scope: CiJobScope): Promise => { /** Create a machine, from a snapshot if one is given, and set it up. */ const start = async ( - createStepId: string, + createStep: { id: string; name: string }, options: { name: string; snapshotId?: string; }, ): Promise => { const sandbox = options.snapshotId - ? await tools.create(createStepId, { + ? await tools.create(createStep, { name: options.name, snapshotId: options.snapshotId, }) - : await tools.create(createStepId, { + : await tools.create(createStep, { name: options.name, ...machineConfig, }); - await sandbox.commands.run(`${createStepId}${scopeSeparator}setup`, [ - "/bin/sh", - "-c", - machineSetupScript, - ]); + await sandbox.commands.run( + { + id: `${createStep.id}${scopeSeparator}setup`, + name: traceName.prepareWorkspace, + }, + ["/bin/sh", "-c", machineSetupScript], + ); return { sandbox }; }; @@ -155,9 +194,13 @@ const createMachine = async (scope: CiJobScope): Promise => { run.ci.hooks.activity(run, scope.jobPath, note); - return start(`${stepId}${scopeSeparator}fresh`, { - name: machineName(run.runId, `${scope.path} fresh`), - }); + return start( + { + id: `${stepId}${scopeSeparator}fresh`, + name: traceName.createFreshMachine, + }, + { name: machineName(run.runId, `${scope.path} fresh`) }, + ); }; const parentId = scope.fromJobIds[0] ?? "the parent"; @@ -166,6 +209,9 @@ const createMachine = async (scope: CiJobScope): Promise => { let started: Started; if (snapshotId) { + // Every job starting from a snapshot checks it for itself, so what a job + // does never depends on whether a sibling got there first. Only the + // rebuild of the parent, which each of them awaits, is shared. run.ci.hooks.activity( run, scope.jobPath, @@ -181,7 +227,7 @@ const createMachine = async (scope: CiJobScope): Promise => { let rebuild = { broken: false, unnamed: false }; try { - probed = await start(stepId, { name, snapshotId }); + probed = await start(create, { name, snapshotId }); } catch (error) { if (!isStartFailure(error)) { throw error; @@ -223,10 +269,13 @@ const createMachine = async (scope: CiJobScope): Promise => { // With the default wait, the same as the first try's: a shorter one // would make a slow node more likely to fail the retry, and a second // failure deletes the snapshot. - probed = await start(restartStepId, { - name: restartName, - snapshotId, - }); + probed = await start( + { id: restartStepId, name: traceName.restartMachine }, + { + name: restartName, + snapshotId, + }, + ); bad = undefined; @@ -273,10 +322,13 @@ const createMachine = async (scope: CiJobScope): Promise => { const retryName = machineName(run.runId, `${scope.path} retry`); try { - started = await start(retryStepId, { - name: retryName, - snapshotId: replacement, - }); + started = await start( + { id: retryStepId, name: traceName.retryCreate }, + { + name: retryName, + snapshotId: replacement, + }, + ); } catch (error) { if (!isStartFailure(error)) { throw error; @@ -299,7 +351,7 @@ const createMachine = async (scope: CiJobScope): Promise => { scope.startNote ?? "creating machine…", ); - started = await start(stepId, { name }); + started = await start(create, { name }); } const handle: MachineHandle = { @@ -313,7 +365,8 @@ const createMachine = async (scope: CiJobScope): Promise => { /** * Best-effort cleanup of a machine that failed to start. The platform keeps - * such a machine (stuck in STARTING) and its name, so it is destroyed here. + * such a machine (stuck in STARTING) and its name, so it is destroyed here, and + * the end of the run finds it by its name if this couldn't. * * Never throws: a machine that can't be cleaned up now must not fail the job. */ @@ -330,7 +383,7 @@ const discardFailedStart = async ( await run.step.run( { id: `${stepId}${scopeSeparator}discard`, - name: `${stepId}${scopeSeparator}discard`, + name: traceName.discardMachine, }, async (): Promise<{ id?: string }> => { // Errors are swallowed inside the step so it never retries. @@ -364,7 +417,7 @@ const discardFailedStart = async ( await sandbox?.destroy(); } } catch { - // Left to expire with the rest of the run's machines. + // The run's cleanup finds it again by its name. } return id ? { id } : {}; @@ -433,6 +486,14 @@ const errorStatus = (error: unknown): number | undefined => { return seen?.status ?? seen?.cause?.status; }; +/** Run `fn` in a job's "Save sandbox" span, which holds snapshotting it. */ +const inSaveSpan = (jobPath: string, fn: () => R): R => { + return inSpan( + ciSpan(`${jobPath}${scopeSeparator}save`, traceName.saveMachine), + fn, + ); +}; + /** How a job's snapshot is named. */ export interface SnapshotCache { target: CacheTarget; @@ -462,6 +523,11 @@ export interface TakenSnapshot { reused: boolean; } +/** A snapshot step: its ID is the step's, its name reads as an action. */ +const snapshotStep = (id: string) => { + return { id, name: traceName.snapshotMachine }; +}; + /** * Snapshot a job's own machine, which is the end of a build or a failed job * that keeps its machine. With a cache, the snapshot is named after the job's @@ -483,7 +549,9 @@ export const snapshotMachine = async ( const handle = await scope.machine; - return takeSnapshot(run, scope.path, handle, cache); + return inSaveSpan(scope.path, () => { + return takeSnapshot(run, scope.path, handle, cache); + }); }; const takeSnapshot = async ( @@ -533,7 +601,7 @@ const createNamedSnapshot = async ( const name = target.name; const attempt = async (id: string): Promise => { - const snapshot = await handle.sandbox.snapshot(id, { name }); + const snapshot = await handle.sandbox.snapshot(snapshotStep(id), { name }); return { snapshotId: snapshot.id, @@ -590,7 +658,9 @@ const createNamedSnapshot = async ( ); } - const unnamed = await handle.sandbox.snapshot(`${stepId} (unnamed)`); + const unnamed = await handle.sandbox.snapshot( + snapshotStep(`${stepId} (unnamed)`), + ); // The name was refused or couldn't be freed, so the job's snapshot falls // back to an unnamed one that no later run can find. The run that invoked @@ -608,7 +678,7 @@ const createRunSnapshot = async ( handle: MachineHandle, stepId: string, ): Promise => { - const snapshot = await handle.sandbox.snapshot(stepId); + const snapshot = await handle.sandbox.snapshot(snapshotStep(stepId)); return { snapshotId: snapshot.id, reused: false }; }; @@ -689,10 +759,10 @@ export const destroyRunMachines = async ( attempt = 0, ): Promise => { await run.step.run( - { - id: `pipeline${scopeSeparator}cleanup${attempt > 0 ? ` (attempt ${attempt})` : ""}`, - name: "cleanup", - }, + ciStep( + `pipeline${scopeSeparator}cleanup${attempt > 0 ? ` (attempt ${attempt})` : ""}`, + traceName.cleanUpMachines, + ), async () => { return destroyOrphans(run.ci.client, run.runId); }, @@ -722,10 +792,10 @@ export const deleteRunSnapshots = async ( // Always there, and reads the set when it runs, for the reason cleaning up // machines is. await run.step.run( - { - id: `pipeline${scopeSeparator}cleanup:snapshots${attempt > 0 ? ` (attempt ${attempt})` : ""}`, - name: "cleanup:snapshots", - }, + ciStep( + `pipeline${scopeSeparator}cleanup:snapshots${attempt > 0 ? ` (attempt ${attempt})` : ""}`, + traceName.cleanUpSnapshots, + ), async () => { const deleted: string[] = []; const failed: string[] = []; diff --git a/packages/ci/src/machine/spans.test.ts b/packages/ci/src/machine/spans.test.ts new file mode 100644 index 000000000..681b0df14 --- /dev/null +++ b/packages/ci/src/machine/spans.test.ts @@ -0,0 +1,583 @@ +/** + * Tests of the trace spans CI groups its steps under, and what it names them: + * one span per job, and in it one per machine, one per command, and one per + * attempt for a command with retries, with GitHub check updates in a span of + * their own. Spans and names never change a step's ID. A failed run of a + * command ends in a failing step, so its span shows the failure. CI's own + * work carries CI's origin, and what you wrote carries none. + * + * @module + */ + +import { version as sdkVersion, step } from "inngest"; +import { describe, expect, test } from "vitest"; +import { CommandFailedError } from "../errors.ts"; +import { consoleReporter } from "../github/auth.ts"; +import { createCi } from "../pipeline/createCi.ts"; +import { ciOrigin } from "../pipeline/names.ts"; +import { createCiTestClient } from "../testing/client.ts"; +import { prEvent } from "../testing/events.ts"; +import type { CommandScript, FakeSandboxApi } from "../testing/fakeSandbox.ts"; +import { createFakeSandboxApi } from "../testing/fakeSandbox.ts"; +import { runFunction } from "../testing/runFunction.ts"; +import { version } from "../version.ts"; +import { $ } from "./command.ts"; +import { sandbox } from "./sandbox.ts"; + +type Ci = ReturnType; + +interface TraceNode { + label: string; + children: Map; +} + +/** + * Draw a run's trace as an indented tree, the way the UI nests it: each span + * once, where its first step is, with its kind in brackets, and each step + * under its spans. + */ +const drawTrace = ( + result: Awaited>, + /** Suffix each row with who it says did it, as in `Create sandbox <- ci`. */ + withOrigins = false, +): string => { + const root: TraceNode = { label: "", children: new Map() }; + + const by = (origin: string | undefined) => { + if (!withOrigins || !origin) { + return ""; + } + + const names: Record = { + [ciOrigin]: "ci", + [`inngest@${sdkVersion}`]: "inngest", + }; + + return ` <- ${names[origin] ?? origin}`; + }; + + for (const stepId of result.stepIds) { + let node = root; + + for (const span of result.spans[stepId] ?? []) { + const key = `span:${span.id}`; + const label = span.kind ? `${span.name} [${span.kind}]` : span.name; + + const child = node.children.get(key) ?? { + label: `${label}${by(span.origin)}`, + children: new Map(), + }; + + node.children.set(key, child); + + node = child; + } + + node.children.set(`step:${stepId}`, { + label: `${result.names[stepId] ?? stepId}${by(result.origins[stepId])}`, + children: new Map(), + }); + } + + const lines: string[] = []; + + const draw = (node: TraceNode, depth: number) => { + for (const child of node.children.values()) { + lines.push(`${" ".repeat(depth)}${child.label}`); + + draw(child, depth + 1); + } + }; + + draw(root, 0); + + return lines.join("\n"); +}; + +/** Run the pipeline whose handler `define` makes from a fresh client. */ +const runPipeline = async ( + define: (ci: Ci) => () => Promise, + scripts: CommandScript[] = [], + api: FakeSandboxApi = createFakeSandboxApi(), +) => { + api.script(scripts); + + const ci = createCi(createCiTestClient(api), { + github: consoleReporter(), + }); + + const pipeline = ci.pipeline( + { id: "pr", on: [{ event: "github/pull_request.opened" }] }, + define(ci), + ); + + return runFunction(pipeline, { event: prEvent }); +}; + +/** Run a job with a captured, a managed, a retried and a background command. */ +const run = () => { + return runPipeline( + (ci) => { + return ci.job("test", async () => { + await $`quick`.timeout("30s"); + + await $`slow`; + + try { + await $`flaky`.retries(1); + } catch { + // Both attempts fail. + } + + const server = await $`serve`.background(); + + await server.output(); + + await server.exited(); + + await server.kill(); + }); + }, + [ + { match: "slow", ticks: 1, stdout: "done" }, + { match: "flaky", exitCode: 1 }, + { match: "serve", ticks: 1, stdout: "listening" }, + ], + ); +}; + +/** + * A pipeline with a row of every sort: a job started from another, plain and + * labelled commands, a step of your own and an extra sandbox. + */ +const rowsPipeline = (ci: Ci) => { + const base = ci.job( + { id: "base", name: "Install dependencies" }, + async () => { + await $`pnpm install`; + }, + ); + + return ci.job({ id: "test", from: base }, async () => { + await $`pnpm test`; + + await $`pnpm exec eslint .`.as("lint"); + + await step.run("my-step", () => { + return "done"; + }); + + const api = await sandbox("api"); + + await api.$`pnpm start`.timeout("30s"); + }); +}; + +/** The span of a command `test` ran. */ +const command = (label: string) => { + return { id: `test › ${label}`, name: `$ ${label}` }; +}; + +const origin = ciOrigin; +const github = [{ id: "github", name: "GitHub", origin }]; +const job = { id: "test", name: "test", kind: "job" }; +const machine = [job, { id: "test › machine", name: "Start sandbox", origin }]; +const quick = [job, command("quick")]; +const slow = [job, command("slow")]; +const attempt1 = [ + job, + command("flaky"), + { id: "attempt-1", name: "Attempt 1", origin }, +]; +const attempt2 = [ + job, + command("flaky"), + { id: "attempt-2", name: "Attempt 2", origin }, +]; +const serve = [job, command("serve")]; + +describe("spans", () => { + test("group each job, machine, command and retried attempt", async () => { + const result = await run(); + + expect(result.type).toBe("function-resolved"); + + // Background calls re-enter their command's span. Check updates are in + // the run's GitHub span, and cleanup is in none. + expect(result.spans).toEqual({ + "github › check:pr:start": github, + "github › check:test:start": github, + "test › machine": machine, + "test › machine › setup": machine, + "test › quick": quick, + "test › slow › start": slow, + "test › slow › wait #1": slow, + "test › slow › check #1": slow, + "test › slow › wait #2": slow, + "test › slow › check #2": slow, + "test › slow › output": slow, + "test › flaky #attempt-1 › start": attempt1, + "test › flaky #attempt-1 › wait #1": attempt1, + "test › flaky #attempt-1 › check #1": attempt1, + "test › flaky #attempt-1 › output": attempt1, + "test › flaky #attempt-1 › exit": attempt1, + "github › check:test:attempt:1": github, + "test › flaky #attempt-2 › start": attempt2, + "test › flaky #attempt-2 › wait #1": attempt2, + "test › flaky #attempt-2 › check #1": attempt2, + "test › flaky #attempt-2 › output": attempt2, + "test › flaky #attempt-2 › exit": attempt2, + "test › serve › start": serve, + "test › serve › output #1": serve, + "test › serve › wait #2": serve, + "test › serve › check #2": serve, + "test › serve › wait #3": serve, + "test › serve › check #3": serve, + "test › serve › output": serve, + "test › serve › kill": serve, + "github › check:test:complete": github, + "github › check:jobs:complete": github, + "github › check:pr:complete": github, + }); + }); + + test("leave step IDs and their order as they were, plus one per failed run", async () => { + const result = await run(); + + expect(result.stepIds).toEqual([ + "github › check:pr:start", + "github › check:test:start", + "test › machine", + "test › machine › setup", + "test › quick", + "test › slow › start", + "test › slow › wait #1", + "test › slow › check #1", + "test › slow › wait #2", + "test › slow › check #2", + "test › slow › output", + "test › flaky #attempt-1 › start", + "test › flaky #attempt-1 › wait #1", + "test › flaky #attempt-1 › check #1", + "test › flaky #attempt-1 › output", + "test › flaky #attempt-1 › exit", + "github › check:test:attempt:1", + "test › flaky #attempt-2 › start", + "test › flaky #attempt-2 › wait #1", + "test › flaky #attempt-2 › check #1", + "test › flaky #attempt-2 › output", + "test › flaky #attempt-2 › exit", + "test › serve › start", + "test › serve › output #1", + "test › serve › wait #2", + "test › serve › check #2", + "test › serve › wait #3", + "test › serve › check #3", + "test › serve › output", + "test › serve › kill", + "github › check:test:complete", + "github › check:jobs:complete", + "github › check:pr:complete", + "pipeline › cleanup", + "pipeline › cleanup:snapshots", + ]); + }); + + test("name each step for what it does, not for what its spans show", async () => { + const result = await run(); + + expect(result.names).toMatchObject({ + "github › check:pr:start": "Create check: pr", + "github › check:test:start": "Report test: started", + "test › machine": "Create sandbox", + "test › machine › setup": "Prepare workspace", + "test › quick": "Run and read output", + "test › slow › start": "Start process", + "test › slow › wait #1": "Wait 1s", + "test › slow › check #1": "Poll process", + "test › slow › wait #2": "Wait 2s", + "test › slow › output": "Read output", + "github › check:test:attempt:1": "Report test: retrying (attempt 2 of 2)", + "test › serve › output #1": "Read output", + "test › serve › kill": "Stop process", + "github › check:test:complete": "Report test: passed", + "github › check:jobs:complete": "Report jobs: ended with the run", + "github › check:pr:complete": "Complete check: pr", + "pipeline › cleanup": "Clean up sandboxes", + }); + }); + + test("end a failed run's span in a failing step named for its exit", async () => { + const result = await run(); + + const attempt1Steps = result.stepIds.filter((id) => { + return id.startsWith("test › flaky #attempt-1"); + }); + + expect(attempt1Steps.at(-1)).toBe("test › flaky #attempt-1 › exit"); + + expect(result.spans["test › flaky #attempt-1 › exit"]).toEqual(attempt1); + + expect(result.names["test › flaky #attempt-1 › exit"]).toBe( + "Exited with code 1", + ); + + expect(result.steps["test › flaky #attempt-1 › exit"]).toMatchObject({ + name: "NonRetriableError", + message: "exit 1", + }); + }); + + test("record nothing more for a passing or `.nothrow()` command", async () => { + const result = await runPipeline( + (ci) => { + return ci.job("test", async () => { + await $`pass`; + + await $`fail`.nothrow(); + }); + }, + [{ match: "fail", exitCode: 1 }], + ); + + expect(result.type).toBe("function-resolved"); + + expect(result.stepIds).toEqual([ + "github › check:pr:start", + "github › check:test:start", + "test › machine", + "test › machine › setup", + "test › pass › start", + "test › pass › wait #1", + "test › pass › check #1", + "test › pass › output", + "test › fail › start", + "test › fail › wait #1", + "test › fail › check #1", + "test › fail › output", + "github › check:test:complete", + "github › check:jobs:complete", + "github › check:pr:complete", + "pipeline › cleanup", + "pipeline › cleanup:snapshots", + ]); + }); + + test("still retry after a failed run, and throw the same error at the end", async () => { + let caught: unknown; + + const result = await runPipeline( + (ci) => { + return ci.job("test", async () => { + try { + await $`flaky`.retries(1); + } catch (error) { + caught = error; + } + }); + }, + [{ match: "flaky", exitCode: 1, stdout: "out", stderr: "flake" }], + ); + + expect(result.type).toBe("function-resolved"); + + expect(result.stepIds).toContain("test › flaky #attempt-2 › exit"); + + expect(caught).toBeInstanceOf(CommandFailedError); + + expect(caught).toMatchObject({ + message: "`flaky` exited with 1\nflake", + command: ["flaky"], + exitCode: 1, + stdoutTail: "out", + stderrTail: "flake", + jobPath: "test", + }); + }); + + test("keep the build of a parent outside every job's span", async () => { + const result = await runPipeline((ci) => { + const base = ci.job("base", async () => { + await $`install`; + }); + + return ci.job({ id: "test", from: base }, async () => { + await $`unit`; + }); + }); + + expect(result.type).toBe("function-resolved"); + + const jobOf = Object.fromEntries( + Object.entries(result.spans).map(([stepId, spans]) => { + return [stepId, spans[0]?.id]; + }), + ); + + // `base` runs in a run of its own, so the pipeline has only the steps that + // look it up and invoke it, outside any job whichever child asks first. + // Check updates are in the GitHub span. + expect(result.stepIds).toContain("base (from) › lookup"); + expect(result.stepIds).toContain("base (from) › build"); + expect(jobOf).toEqual({ + "github › check:pr:start": "github", + "github › check:test:start": "github", + "test › machine": "test", + "test › machine › setup": "test", + "test › unit › start": "test", + "test › unit › wait #1": "test", + "test › unit › check #1": "test", + "test › unit › output": "test", + "github › check:test:complete": "github", + "github › check:jobs:complete": "github", + "github › check:pr:complete": "github", + }); + }); + + test("name a pipeline's rows for what they are", async () => { + const result = await runPipeline(rowsPipeline); + + expect(result.type).toBe("function-resolved"); + + expect(drawTrace(result)).toBe( + [ + "GitHub", + " Create check: pr", + " Report test: started", + " Report test: passed", + " Report jobs: ended with the run", + " Complete check: pr", + "Look up cache", + "Build base in its own run", + "test [job]", + " Start sandbox from base", + " Create sandbox", + " Prepare workspace", + " $ pnpm test", + " Start process", + " Wait 1s", + " Poll process", + " Read output", + " lint", + " Start process", + " Wait 1s", + " Poll process", + " Read output", + " my-step", + " api", + " Start sandbox", + " Create sandbox", + " Prepare workspace", + " $ pnpm start", + " Run and read output", + "Clean up sandboxes", + "Clean up snapshots", + ].join("\n"), + ); + }); + + test("mark the work CI does for you, and nothing you wrote", async () => { + const result = await runPipeline(rowsPipeline); + + expect(result.type).toBe("function-resolved"); + + expect(ciOrigin).toBe(`@inngest/ci@${version}`); + + // Jobs, commands, `my-step` and the `api` sandbox are yours. `snapshot()` + // opens its span without an origin, so it inherits the one of the CI span + // it runs in, while its own steps are marked by the SDK. + expect(drawTrace(result, true)).toBe( + [ + "GitHub <- ci", + " Create check: pr <- ci", + " Report test: started <- ci", + " Report test: passed <- ci", + " Report jobs: ended with the run <- ci", + " Complete check: pr <- ci", + "Look up cache <- ci", + "Build base in its own run <- ci", + "test [job]", + " Start sandbox from base <- ci", + " Create sandbox <- ci", + " Prepare workspace <- ci", + " $ pnpm test", + " Start process <- ci", + " Wait 1s <- ci", + " Poll process <- ci", + " Read output <- ci", + " lint", + " Start process <- ci", + " Wait 1s <- ci", + " Poll process <- ci", + " Read output <- ci", + " my-step", + " api", + " Start sandbox <- ci", + " Create sandbox <- ci", + " Prepare workspace <- ci", + " $ pnpm start", + " Run and read output <- ci", + "Clean up sandboxes <- ci", + "Clean up snapshots <- ci", + ].join("\n"), + ); + }); + + test("mark every step a command runs, through retries and background calls", async () => { + const result = await run(); + + expect(result.type).toBe("function-resolved"); + + // The job only runs commands, so every step in it is CI's. + for (const stepId of result.stepIds) { + expect(result.origins[stepId]).toBe(ciOrigin); + } + }); + + test("ask for a parent's snapshot again, as a build of its own, when it won't start", async () => { + const api = createFakeSandboxApi(); + const { fetch } = api; + let broken = false; + + // Break the snapshot as soon as it's taken, before `test` starts from it. + api.fetch = (input, init) => { + if (!broken && api.snapshots.size > 0) { + broken = true; + + api.failSnapshotStarts(); + } + + return fetch(input, init); + }; + + const result = await runPipeline( + (ci) => { + const parent = ci.job("base", async () => { + await $`pnpm install`; + }); + + return ci.job({ id: "test", from: parent }, async () => { + await $`pnpm test`; + }); + }, + [], + api, + ); + + expect(result.type).toBe("function-resolved"); + + const builds = result.stepIds.filter((id) => { + return id.endsWith("› build"); + }); + + expect(builds).toEqual([ + "base (from) › build", + "base (from) (rebuild) › build", + ]); + + // Neither is in a job's span. + for (const id of builds) { + expect(result.spans[id]?.[0]).toBeUndefined(); + } + }); +}); diff --git a/packages/ci/src/pipeline/README.md b/packages/ci/src/pipeline/README.md index 42650525b..d6748126e 100644 --- a/packages/ci/src/pipeline/README.md +++ b/packages/ci/src/pipeline/README.md @@ -10,5 +10,7 @@ The client and the run engine. - `matrix.ts`: `ci.matrix()`, matrix expansion and the concurrency pool. - `metadata.ts`: the `userland.inngest-ci` metadata attached to runs and steps, and the helpers that build and attach it. - `scope.ts`: the run and job scopes held in async context. Jobs only write to the run scope, except for `builds`, the promises of the builds `from` parents need. +- `names.ts`: what each step and span CI writes is called in the trace, and the origin that marks CI's own work. +- `spans.ts`: the one place CI touches the SDK's experimental trace-span API. It is feature-detected, so CI runs unchanged on an SDK without it. - `durable.ts`: durable proxies that run calls as steps. - `rerun.ts`: re-running a pipeline from a GitHub check. diff --git a/packages/ci/src/pipeline/cacheGuarantees.test.ts b/packages/ci/src/pipeline/cacheGuarantees.test.ts index d1b05bb45..c34aa0b88 100644 --- a/packages/ci/src/pipeline/cacheGuarantees.test.ts +++ b/packages/ci/src/pipeline/cacheGuarantees.test.ts @@ -471,6 +471,18 @@ describe("jobs that start from one parent", () => { expect(buildSteps(result.stepIds)).toEqual(["install (from) › build"]); expect(count(api, "pnpm install")).toBe(1); + + for (const job of ["a", "b"]) { + const inJob = result.stepIds.filter((stepId) => { + return result.spans[stepId]?.[0]?.id === job; + }); + + expect(inJob[0]).toBe(`${job} › machine`); + expect(result.spans[`${job} › machine`]?.[0]?.name).toBe(job); + } + + // Many jobs share the build, so it sits in none of them. + expect(result.spans["install (from) › build"] ?? []).toEqual([]); }); test("a warm hit invokes no build, and the parent is still looked up once", async () => { diff --git a/packages/ci/src/pipeline/durable.ts b/packages/ci/src/pipeline/durable.ts index b8971aab1..757ef95fa 100644 --- a/packages/ci/src/pipeline/durable.ts +++ b/packages/ci/src/pipeline/durable.ts @@ -178,7 +178,8 @@ const call = async ( // inside the handler; that's how the call knows which repository it's for. const step = getRunScope()?.step ?? execution.ctx.step; - return step.run({ id, name: overrides.name ?? id }, invoke); + // Named by the method called, without the job path or count in its ID. + return step.run({ id, name: overrides.name ?? label }, invoke); }; const invokeOnClient = async ( diff --git a/packages/ci/src/pipeline/job.ts b/packages/ci/src/pipeline/job.ts index 8528e9c71..447587df6 100644 --- a/packages/ci/src/pipeline/job.ts +++ b/packages/ci/src/pipeline/job.ts @@ -33,9 +33,11 @@ import type { AnyJob, CheckConclusion, JobConfig } from "../types.ts"; import { errorMessage, formatDuration, shortReason } from "../util.ts"; import type { CacheBuildData, CacheBuildResult } from "./cacheBuild.ts"; import { tagStep } from "./metadata.ts"; +import { ciStep, traceName } from "./names.ts"; import type { CiJobScope, CiRunScope } from "./scope.ts"; import { getRunScope, + inJobSpan, matrixOriginOf, rootRunIdOf, runJobBody, @@ -231,10 +233,11 @@ export const invokeBuild = async ({ let output: CacheBuildResult | null; try { - const buildStepId = `${stepPath}${scopeSeparator}build`; - output = (await run.step.invoke( - { id: buildStepId, name: buildStepId }, + ciStep( + `${stepPath}${scopeSeparator}build`, + traceName.buildInOwnRun(path), + ), { function: run.ci.cacheBuild(), data }, )) as CacheBuildResult | null; } catch (error) { @@ -394,7 +397,16 @@ export const validateInput = async ( ); }; -const jobBody = async ({ +/** Everything a job does is in its span. */ +const jobBody = ( + args: RunJobArgs & { run: CiRunScope; path: string; number: number }, +): Promise => { + return inJobSpan(args.run, args.path, () => { + return jobSteps(args); + }); +}; + +const jobSteps = async ({ run, config: declared, handler, @@ -536,7 +548,12 @@ const jobBody = async ({ const checkStartedAt = checked ? await checks.jobStart(target) : undefined; const startedAt = checkStartedAt ?? - (await durableNow(run, `start:${scope.path}`, scope.path)); + (await durableNow( + run, + `start:${scope.path}`, + traceName.recordStartTime, + scope.path, + )); if (checked) { run.openChecks.set(scope.path, checkName); @@ -614,7 +631,13 @@ const jobBody = async ({ } const endedAt = - checkEndedAt ?? (await durableNow(run, `end:${scope.path}`, scope.path)); + checkEndedAt ?? + (await durableNow( + run, + `end:${scope.path}`, + traceName.recordEndTime, + scope.path, + )); const durationMs = endedAt - startedAt; run.summaries.push({ @@ -658,7 +681,13 @@ const jobBody = async ({ } const endedAt = - checkEndedAt ?? (await durableNow(run, `end:${scope.path}`, scope.path)); + checkEndedAt ?? + (await durableNow( + run, + `end:${scope.path}`, + traceName.recordEndTime, + scope.path, + )); run.summaries.push({ path: scope.path, @@ -679,9 +708,10 @@ const jobBody = async ({ const durableNow = ( run: CiRunScope, id: string, + name: string, jobPath: string, ): Promise => { - return run.step.run({ id, name: id }, async () => { + return run.step.run(ciStep(id, name), async () => { await tagStep(run, { kind: "job", job: jobPath }); return Date.now(); diff --git a/packages/ci/src/pipeline/metadata.test.ts b/packages/ci/src/pipeline/metadata.test.ts index 3984c8ee8..a9e6571e2 100644 --- a/packages/ci/src/pipeline/metadata.test.ts +++ b/packages/ci/src/pipeline/metadata.test.ts @@ -19,6 +19,7 @@ import { runFunction } from "../testing/runFunction.ts"; import { version } from "../version.ts"; import { createCi } from "./createCi.ts"; import { tagStep } from "./metadata.ts"; +import { ciOrigin } from "./names.ts"; import type { CiRunScope } from "./scope.ts"; const setup = () => { @@ -250,6 +251,9 @@ describe("run metadata", () => { expect(result.stepIds).toContain("ci › metadata:start"); expect(result.stepIds).toContain("ci › metadata:end"); + + expect(result.origins["ci › metadata:start"]).toBe(ciOrigin); + expect(result.origins["ci › metadata:end"]).toBe(ciOrigin); }); test("a pipeline with its check on adds no steps of its own", async () => { @@ -306,6 +310,11 @@ describe("step metadata", () => { expect(update.kind).toBe("userland.inngest-ci"); expect(update.op).toBe("merge"); } + + // Caching, start times and checks are all CI's work, not the jobs'. + for (const [step] of tags) { + expect(result.origins[step as string]).toBe(ciOrigin); + } }); }); diff --git a/packages/ci/src/pipeline/metadata.ts b/packages/ci/src/pipeline/metadata.ts index ce6a8986d..72f2c1657 100644 --- a/packages/ci/src/pipeline/metadata.ts +++ b/packages/ci/src/pipeline/metadata.ts @@ -11,6 +11,7 @@ import { getAsyncCtx } from "inngest/experimental"; import type { CheckConclusion } from "../types.ts"; import { version } from "../version.ts"; +import { ciStep, traceName } from "./names.ts"; import type { CiRunScope } from "./scope.ts"; import { apiNames } from "./scope.ts"; @@ -149,7 +150,7 @@ export const metadataStep = ( id: string, runValues: () => Record, ): Promise => { - return run.step.run({ id, name: id }, async () => { + return run.step.run(ciStep(id, traceName.recordRunDetails), async () => { await tagStep(run, undefined, runValues()); return Date.now(); diff --git a/packages/ci/src/pipeline/names.ts b/packages/ci/src/pipeline/names.ts new file mode 100644 index 000000000..82964bf51 --- /dev/null +++ b/packages/ci/src/pipeline/names.ts @@ -0,0 +1,160 @@ +/** + * What every step and span CI writes is called in the trace. Rows you wrote + * read the way you wrote them, and CI's own work reads as a plain-English + * action. Names are for reading only: step IDs, which replays depend on, are + * built where each step is and never from a name. + * + * CI's own work is also marked with CI's origin, so a trace can set it apart + * from what you wrote. Rows you wrote are never marked. + * + * @module + */ + +import type { CheckConclusion } from "../types.ts"; +import { formatDuration } from "../util.ts"; +import { version } from "../version.ts"; +import type { CiRunScope } from "./scope.ts"; +import { scopeSeparator } from "./scope.ts"; +import { originOption } from "./spans.ts"; + +/** + * Marks a step or span as work CI does for you rather than something you + * wrote, as the SDK's experimental step origin and span `origin`. + * Steps inherit it from the innermost span that has one, so marking a span + * marks everything in it. + */ +export const ciOrigin = `@inngest/ci@${version}`; + +/** A span CI opens for its own work, such as starting a sandbox. */ +export const ciSpan = (id: string, name: string) => { + return { id, name, origin: ciOrigin }; +}; + +/** + * A step CI runs for you where no span of CI's marks it, such as a command's + * `Start process` inside the command's span, which is yours. + */ +export const ciStep = (id: string, name: string) => { + return { id, name, ...originOption(ciOrigin) }; +}; + +/** How a finished check reads, as in `Report test: passed`. */ +const outcomes: Partial> = { + success: "passed", + failure: "failed", + timed_out: "timed out", +}; + +export const traceName = { + /** A job's span: its `name`, or its path when it has none. */ + job: (run: CiRunScope, path: string): string => { + return run.ci.jobs.get(path)?.config.name ?? path; + }, + + /** The span a job's sandbox starts in. */ + startMachine: (parent?: string): string => { + return parent ? `Start sandbox from ${parent}` : "Start sandbox"; + }, + + /** The span of an extra sandbox, named as `sandbox()` was. */ + extraMachine: (path: string, jobPath: string): string => { + return path.slice(jobPath.length + scopeSeparator.length); + }, + + /** A command's span: its `.as()` label, or the command itself. */ + command: (text: string, label?: string): string => { + return label ?? `$ ${text}`; + }, + + attempt: (attempt: number): string => { + return `Attempt ${attempt}`; + }, + + wait: (ms: number): string => { + return `Wait ${formatDuration(ms)}`; + }, + + exited: (exitCode: number): string => { + return `Exited with code ${exitCode}`; + }, + + buildInOwnRun: (path: string): string => { + return `Build ${path} in its own run`; + }, + + createCheck: (check: string): string => { + return `Create check: ${check}`; + }, + + completeCheck: (check: string): string => { + return `Complete check: ${check}`; + }, + + /** A check update, as in `Report test: started`. */ + report: (subject: string, status: string): string => { + return `Report ${subject}: ${status}`; + }, + + retrying: (attempt: number, of: number): string => { + return `retrying (attempt ${attempt} of ${of})`; + }, + + outcome: (conclusion: CheckConclusion): string => { + return outcomes[conclusion] ?? conclusion; + }, + + /** A `github.*` helper you called, by its method name. */ + githubHelper: (helper: string): string => { + return `github.${helper}`; + }, + + canUser: (login: string, permission: string): string => { + return `Check ${login} can ${permission}`; + }, + + waitForCheck: (name: string): string => { + return `Wait for check: ${name}`; + }, + + waitForWorkflow: (workflow: string): string => { + return `Wait for workflow: ${workflow}`; + }, + + github: "GitHub", + resolveRepository: "Resolve repository", + commentNotAllowed: "Comment: not allowed", + + createMachine: "Create sandbox", + restartMachine: "Restart sandbox", + createFreshMachine: "Create sandbox (fresh)", + retryCreate: "Retry create", + discardMachine: "Discard sandbox", + prepareWorkspace: "Prepare workspace", + saveMachine: "Save sandbox", + snapshotMachine: "Snapshot sandbox", + cleanUpMachines: "Clean up sandboxes", + cleanUpSnapshots: "Clean up snapshots", + + /** A captured command's one step: it runs and returns its output at once. */ + runAndReadOutput: "Run and read output", + startProcess: "Start process", + pollProcess: "Poll process", + readOutput: "Read output", + stopProcess: "Stop process", + stopAfterTimeout: "Stop after timeout", + findStartedProcess: "Find started process", + + checkCache: "Check cache", + lookUpCache: "Look up cache", + resolveCacheName: "Resolve cache name", + deleteBadSnapshot: "Delete bad snapshot", + checkSnapshotState: "Check snapshot state", + + findChangedFiles: "Find changed files", + + recordStartTime: "Record start time", + recordEndTime: "Record end time", + recordRunDetails: "Record run details", + addSummary: "Add summary", + addAnnotations: "Add annotations", +}; diff --git a/packages/ci/src/pipeline/pipeline.ts b/packages/ci/src/pipeline/pipeline.ts index 1c3bd4c25..4b5c9ce63 100644 --- a/packages/ci/src/pipeline/pipeline.ts +++ b/packages/ci/src/pipeline/pipeline.ts @@ -49,10 +49,12 @@ import type { CacheBuildData } from "./cacheBuild.ts"; import type { RegisteredJob } from "./job.ts"; import { conclusionForError, runJob } from "./job.ts"; import { metadataStep, runEndMetadata, runStartMetadata } from "./metadata.ts"; +import { ciStep, traceName } from "./names.ts"; import type { CiInternals, CiRunScope } from "./scope.ts"; import { apiNames, countApi, + inGitHubSpan, initCiAls, runInScope, withScopePreserved, @@ -566,38 +568,41 @@ const resolveConfiguredRepo = ( ): Promise => { const { owner, name } = parseRepo(fullName); - return run.step.run( - { id: `github › repo:resolve`, name: "repo:resolve" }, - async (): Promise => { - const base: RepoContext = { owner, name, fullName, sha: "" }; - const provider = run.ci.github as GitHubProvider; - - if (provider.kind === "console") { - return base; - } - - const octokit = await provider.octokit({ owner, repo: name }); - const { data: info } = await octokit.rest.repos.get({ - owner, - repo: name, - }); - const branch = info.default_branch; + return inGitHubSpan(run, () => { + return run.step.run( + { id: `github › repo:resolve`, name: traceName.resolveRepository }, + async (): Promise => { + const base: RepoContext = { owner, name, fullName, sha: "" }; + const provider = run.ci.github as GitHubProvider; + + if (provider.kind === "console") { + return base; + } + + const octokit = await provider.octokit({ owner, repo: name }); + const { data: info } = await octokit.rest.repos.get({ + owner, + repo: name, + }); + const branch = info.default_branch; - const { data: head } = await octokit.rest.repos.getBranch({ - owner, - repo: name, - branch, - }); + const { data: head } = await octokit.rest.repos.getBranch({ + owner, + repo: name, + branch, + }); - return { - ...base, - sha: head.commit.sha, - ref: `refs/heads/${branch}`, - baseRef: branch, - trigger: (run.event as { name?: string } | undefined)?.name ?? "manual", - }; - }, - ) as Promise; + return { + ...base, + sha: head.commit.sha, + ref: `refs/heads/${branch}`, + baseRef: branch, + trigger: + (run.event as { name?: string } | undefined)?.name ?? "manual", + }; + }, + ) as Promise; + }); }; /** @@ -826,19 +831,21 @@ const checkCommentPermission = async ( const allowed = await canUser(login, minPermission); if (!allowed) { - await run.step.run( - { id: "github › comment:denied", name: "comment:denied" }, - async () => { - const { stickyComment } = await import("../github/helpers.ts"); - - await stickyComment( - "permission", - `@${login} you need \`${minPermission}\` permission to run \`${commentBody(run.event as { data?: unknown }).split(" ")[0]}\`.`, - ); - - return { denied: login }; - }, - ); + await inGitHubSpan(run, () => { + return run.step.run( + { id: "github › comment:denied", name: traceName.commentNotAllowed }, + async () => { + const { stickyComment } = await import("../github/helpers.ts"); + + await stickyComment( + "permission", + `@${login} you need \`${minPermission}\` permission to run \`${commentBody(run.event as { data?: unknown }).split(" ")[0]}\`.`, + ); + + return { denied: login }; + }, + ); + }); } return allowed; @@ -892,9 +899,12 @@ export const cleanupFunction = ({ async ({ event, step }: any) => { const runId = event?.data?.run_id ?? event?.data?.runId; - return step.run("destroy-orphans", async () => { - return runId ? destroyOrphans(client, runId) : { destroyed: 0 }; - }); + return step.run( + ciStep("destroy-orphans", traceName.cleanUpMachines), + async () => { + return runId ? destroyOrphans(client, runId) : { destroyed: 0 }; + }, + ); }, ); }; diff --git a/packages/ci/src/pipeline/scope.ts b/packages/ci/src/pipeline/scope.ts index 36ee0c732..653f1c2ba 100644 --- a/packages/ci/src/pipeline/scope.ts +++ b/packages/ci/src/pipeline/scope.ts @@ -7,7 +7,7 @@ import type { GetStepTools, Inngest, InngestFunction } from "inngest"; import type { AsyncContext, DurableSandboxTools } from "inngest/experimental"; -import { runWithAsyncCtx } from "inngest/experimental"; +import { getAsyncCtx, runWithAsyncCtx } from "inngest/experimental"; import type { CachedSnapshot } from "../cache/cache.ts"; import { CiUsageError } from "../errors.ts"; import type { @@ -19,6 +19,8 @@ import type { } from "../types.ts"; import type { CacheBuildData, CacheBuildResult } from "./cacheBuild.ts"; import type { CiHooks } from "./hooks.ts"; +import { ciSpan, traceName } from "./names.ts"; +import { inSpan } from "./spans.ts"; /** * The separator used between parts of a scope path and a step label. It's a @@ -510,9 +512,16 @@ export const withScopePreserved = (tools: T): T => { return withStepIdPrefix(tools); }; +/** + * Prefix a step's ID with its scope path, and name it by the ID it was given, + * since its job's span already shows the rest. + */ const prefixStepId = (idOrOptions: unknown, prefix: string): unknown => { if (typeof idOrOptions === "string") { - return `${prefix}${scopeSeparator}${idOrOptions}`; + return { + id: `${prefix}${scopeSeparator}${idOrOptions}`, + name: idOrOptions, + }; } if ( @@ -533,17 +542,57 @@ const prefixStepId = (idOrOptions: unknown, prefix: string): unknown => { return idOrOptions; }; +/** + * Run `fn` in a job's trace span. The span is top-level wherever the job is + * started from, so work on its machine that another job asks for later, like + * a snapshot, comes back to it. + */ +export const inJobSpan = ( + run: CiRunScope, + jobPath: string, + fn: () => R, +): R => { + return runWithAsyncCtx(run.asyncCtx, () => { + return inSpan( + { id: jobPath, name: traceName.job(run, jobPath), kind: "job" }, + fn, + ); + }); +}; + +/** + * Run `fn` outside every job's span, wherever it's called from. Jobs that + * start from one parent share its build, so the build's steps belong to none + * of them, and which one asks first must not decide where they sit in the + * trace. + */ +export const outsideJobs = (run: CiRunScope, fn: () => R): R => { + return runWithAsyncCtx(run.asyncCtx, fn); +}; + +/** + * Run `fn` in the run's one top-level GitHub span, wherever it's called from. + * Reporting isn't a job's own work, so a job's span ends with the job, and a + * failed job doesn't end on a green check update. + */ +export const inGitHubSpan = (run: CiRunScope, fn: () => R): R => { + return runWithAsyncCtx(run.asyncCtx, () => { + return inSpan(ciSpan("github", traceName.github), fn); + }); +}; + /** * Run a job handler inside both the CI job scope and a copy of the SDK's async - * context whose `step` prefixes IDs with the job path. + * context whose `step` prefixes IDs with the job path. It copies the context + * `inJobSpan` set, so the handler's steps are in the job's span. */ export const runJobBody = async ( scope: CiJobScope, fn: () => Promise, ): Promise => { - const { asyncCtx } = scope.run; + const asyncCtx = await getAsyncCtx(); - if (!asyncCtx.execution) { + if (!asyncCtx?.execution) { return runInScope({ run: scope.run, job: scope }, fn); } diff --git a/packages/ci/src/pipeline/spans.test.ts b/packages/ci/src/pipeline/spans.test.ts new file mode 100644 index 000000000..47fd17a5d --- /dev/null +++ b/packages/ci/src/pipeline/spans.test.ts @@ -0,0 +1,146 @@ +/** + * Tests of the span helpers, and of CI on an SDK without the span API: with + * none, functions still run, step IDs and names are exactly what they are + * with it, and the trace just has no spans or origins. The rest of the suite + * runs with the test stub in place. + * + * @module + */ + +import { group, step } from "inngest"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { consoleReporter } from "../github/auth.ts"; +import { $ } from "../machine/command.ts"; +import { createCiTestClient } from "../testing/client.ts"; +import { prEvent } from "../testing/events.ts"; +import { createFakeSandboxApi } from "../testing/fakeSandbox.ts"; +import { runFunction } from "../testing/runFunction.ts"; +import { installSpanStub, removeSpanStub } from "../testing/spanStub.ts"; +import { createCi } from "./createCi.ts"; +import { hasSpanApi, inSpan, originOption } from "./spans.ts"; + +/** Run a job with a command and a step of your own. */ +const run = () => { + const ci = createCi(createCiTestClient(createFakeSandboxApi()), { + github: consoleReporter(), + }); + + const job: () => Promise = ci.job("test", async () => { + await $`pnpm test`; + + await step.run("mine", () => { + return "done"; + }); + }); + + const pipeline = ci.pipeline( + { id: "pr", on: [{ event: "github/pull_request.opened" }] }, + job, + ); + + return runFunction(pipeline, { event: prEvent }); +}; + +describe("without the SDK's span API", () => { + beforeEach(() => { + removeSpanStub(); + }); + + afterEach(() => { + installSpanStub(); + }); + + test("is detected as absent", () => { + expect(hasSpanApi()).toBe(false); + + expect("~span" in group).toBe(false); + }); + + test("inSpan runs the function and returns what it returns", async () => { + const span = { id: "a", name: "A" }; + + expect(inSpan(span, () => 7)).toBe(7); + + await expect( + inSpan(span, () => { + return Promise.resolve("later"); + }), + ).resolves.toBe("later"); + + expect(() => { + return inSpan(span, () => { + throw new Error("from fn"); + }); + }).toThrow("from fn"); + }); + + test("steps keep their IDs and names, and the trace has no spans", async () => { + installSpanStub(); + + const withSpans = await run(); + + removeSpanStub(); + + const without = await run(); + + expect(without.type).toBe("function-resolved"); + + expect(without.stepIds).toEqual(withSpans.stepIds); + expect(without.names).toEqual(withSpans.names); + + expect(without.stepIds).toContain("test › pnpm test › start"); + expect(without.names["test › pnpm test › start"]).toBe("Start process"); + + expect(Object.keys(withSpans.spans).length).toBeGreaterThan(0); + expect(without.spans).toEqual({}); + + // Origins aren't asserted absent: an SDK with the span API stamps a + // step's `~origin` option itself, whether or not `group["~span"]` is there + // to open spans, and this suite runs against that SDK. + }); +}); + +describe("with the SDK's span API", () => { + test("is detected, and inSpan hands it the span and the function", () => { + expect(hasSpanApi()).toBe(true); + + removeSpanStub(); + + const calls: unknown[] = []; + + Object.defineProperty(group, "~span", { + value: (span: unknown, fn: () => unknown) => { + calls.push(span); + + return `wrapped ${String(fn())}`; + }, + configurable: true, + }); + + try { + const span = { id: "a", name: "A", kind: "job" }; + + expect( + inSpan(span, () => { + return "inner"; + }), + ).toBe("wrapped inner"); + + expect(calls).toEqual([span]); + } finally { + Reflect.deleteProperty(group, "~span"); + + installSpanStub(); + } + }); +}); + +describe("originOption", () => { + test("is an extra key on the step options, and nothing else", () => { + expect({ id: "a", name: "A", ...originOption("x") }).toEqual({ + id: "a", + name: "A", + "~origin": "x", + }); + }); +}); diff --git a/packages/ci/src/pipeline/spans.ts b/packages/ci/src/pipeline/spans.ts new file mode 100644 index 000000000..59872b69f --- /dev/null +++ b/packages/ci/src/pipeline/spans.ts @@ -0,0 +1,70 @@ +/** + * The one place CI touches the SDK's experimental trace-span API. The span + * API ships in newer SDKs than the oldest one CI supports, so it's + * feature-detected here: where the SDK has it, spans and origins mark up the + * trace, and where it doesn't, `inSpan` just runs the function and the origin + * step option is an unknown key the SDK ignores. Step IDs and names never + * depend on it. Everything else in CI goes through these helpers rather than + * reaching for the SDK's span API itself. + * + * @module + */ + +import { group } from "inngest"; + +/** A trace span, as the SDK's span API takes it. */ +export interface SpanInfo { + /** The span's stable ID, from which its rows are grouped. */ + id: string; + /** What the span is called in the trace. */ + name: string; + /** What kind of span it is, as in `job`. */ + kind?: string; + /** Who did the work in it, inherited by the steps and spans inside. */ + origin?: string; +} + +/** The part of the SDK's `group` this file uses, when the SDK has it. */ +interface SpanApi { + "~span"?: (span: SpanInfo, fn: () => R) => R; +} + +const spanApi = (): SpanApi => { + return group as unknown as SpanApi; +}; + +/** Whether the SDK in use has the span API. */ +export const hasSpanApi = (): boolean => { + return typeof spanApi()["~span"] === "function"; +}; + +/** + * Run `fn` in a trace span, or just run it when the SDK has no span API. + */ +export const inSpan = ( + /** The span to run `fn` in. */ + span: SpanInfo, + /** What to run. */ + fn: () => R, +): R => { + const api = spanApi(); + const open = api["~span"]; + + if (typeof open !== "function") { + return fn(); + } + + return open.call(api, span, fn) as R; +}; + +/** + * The step option that marks a step as work `origin` did. A step option is + * just a key on the options object, and an SDK without the span API ignores + * it, so it's safe to always pass. + */ +export const originOption = ( + /** Who did the work. */ + origin: string, +): { "~origin": string } => { + return { "~origin": origin }; +}; diff --git a/packages/ci/src/report.ts b/packages/ci/src/report.ts index 237a10650..c66c13230 100644 --- a/packages/ci/src/report.ts +++ b/packages/ci/src/report.ts @@ -4,6 +4,7 @@ * @module */ +import { traceName } from "./pipeline/names.ts"; import { countApi, getJobScope, @@ -43,7 +44,7 @@ export const report = { const id = nextStepId(run, job?.path, "report:summary"); - await run.step.run({ id, name: id }, () => { + await run.step.run({ id, name: traceName.addSummary }, () => { return { length: markdown.length, }; @@ -94,7 +95,7 @@ export const report = { const id = nextStepId(run, job?.path, "report:annotate"); - await run.step.run({ id, name: id }, () => { + await run.step.run({ id, name: traceName.addAnnotations }, () => { return { count: valid.length, }; diff --git a/packages/ci/src/testing/README.md b/packages/ci/src/testing/README.md index 688f4b139..7893f48ba 100644 --- a/packages/ci/src/testing/README.md +++ b/packages/ci/src/testing/README.md @@ -6,5 +6,6 @@ Test-only helpers. Not exported and not built. - `fakeGitHub.ts`: a fake GitHub HTTP layer. - `client.ts`: an Inngest client wired to the fake sandbox API, which also records the functions it creates. - `events.ts`: the pull request event and trigger tests share. +- `spanStub.ts`, `setup.ts`: a stand-in for the SDK's span API, installed before every test file when the SDK has none, so span and origin assertions hold on any SDK. - `runFunction.ts`: drives a function to completion like the executor, including `step.invoke` of the functions the test client created. Invokes run beside the function, and the function is called again as each one ends; runs of one `event.data` concurrency key take turns. - `schema.ts`: a fake Standard Schema that can also write itself as JSON Schema. diff --git a/packages/ci/src/testing/runFunction.ts b/packages/ci/src/testing/runFunction.ts index 805bee977..52b6f1085 100644 --- a/packages/ci/src/testing/runFunction.ts +++ b/packages/ci/src/testing/runFunction.ts @@ -15,6 +15,7 @@ import type { EventPayload, InngestFunction } from "inngest"; import { createdFunctions } from "./client.ts"; +import { SpanStubMiddleware, spanStubInstalled, stampOf } from "./spanStub.ts"; /** The opcodes the harness reads; their values are the wire format. */ const StepOpCode = { @@ -41,11 +42,19 @@ interface Step { data?: unknown; error?: unknown; metadata?: MetadataUpdate[]; - opts?: unknown; + opts?: { span?: StepSpanPath; origin?: string }; /** The step's ID before hashing. */ userland?: { id: string }; } +/** The spans a step is grouped under, outermost first. */ +type StepSpanPath = Array<{ + id: string; + name: string; + kind?: string; + origin?: string; +}>; + /** One execution request's outcome, loosely typed: the SDK doesn't export it. */ interface ExecutionResult { type: string; @@ -73,6 +82,10 @@ export interface RunResult { names: Record; /** Step data keyed by step ID. */ steps: Record; + /** The span path of each step in a span, keyed by step ID. */ + spans: Record; + /** The origin of each step that has one, keyed by step ID. */ + origins: Record; /** * What each `step.run` step was planned with, keyed by step ID: the options * the SDK sent the executor, such as a sandbox call's input. @@ -288,6 +301,8 @@ export const runFunction = async ( const batches: string[][] = []; const names: Record = {}; const steps: Record = {}; + const spans: RunResult["spans"] = {}; + const origins: RunResult["origins"] = {}; const inputs: Record = {}; const metadata: RunResult["metadata"] = []; @@ -324,6 +339,21 @@ export const runFunction = async ( names[id] = step.displayName ?? step.name ?? id; steps[id] = step.data; + + // Where the SDK has the span API it stamps these on the step. Where the + // test stub provides it, the stub recorded them instead. + const stamp = spanStubInstalled() ? stampOf(id) : undefined; + const span = + step.opts?.span ?? (stamp?.span.length ? stamp.span : undefined); + const origin = step.opts?.origin ?? stamp?.origin; + + if (span) { + spans[id] = span; + } + + if (origin) { + origins[id] = origin; + } }; // The executor retries a step that failed retriably, and only writes the @@ -382,6 +412,7 @@ export const runFunction = async ( ...(planned.displayName === undefined ? {} : { displayName: planned.displayName }), + opts: planned.opts, ...(outcome?.error === undefined ? { data: outcome?.data ?? null } : { error: outcome.error }), @@ -404,6 +435,8 @@ export const runFunction = async ( batches, names, steps, + spans, + origins, inputs, metadata, }; @@ -424,6 +457,8 @@ export const runFunction = async ( batches, names, steps, + spans, + origins, inputs, metadata, }; @@ -490,6 +525,7 @@ export const runFunction = async ( ? {} : { displayName: planned.displayName }), data: opts.resolveWait ? opts.resolveWait(planned) : null, + opts: planned.opts, }); continue; @@ -544,6 +580,11 @@ const runOnce = async ( return new Middleware({ client }); }); + if (spanStubInstalled()) { + // First, so the other middleware builds on step tools that record. + middlewareInstances.unshift(new SpanStubMiddleware({ client })); + } + const execution = internals["createExecution"]({ partialOptions: { client, diff --git a/packages/ci/src/testing/setup.ts b/packages/ci/src/testing/setup.ts new file mode 100644 index 000000000..f7532a0aa --- /dev/null +++ b/packages/ci/src/testing/setup.ts @@ -0,0 +1,11 @@ +/** + * Runs before every test file: gives the SDK the span API when it has none, + * so span and origin assertions hold against any SDK. A test of the no-span + * path takes it off again with `removeSpanStub`. + * + * @module + */ + +import { installSpanStub } from "./spanStub.ts"; + +installSpanStub(); diff --git a/packages/ci/src/testing/spanStub.ts b/packages/ci/src/testing/spanStub.ts new file mode 100644 index 000000000..481979fbe --- /dev/null +++ b/packages/ci/src/testing/spanStub.ts @@ -0,0 +1,185 @@ +/** + * A stand-in for the SDK's experimental trace-span API, for tests that run + * against an SDK that doesn't have it yet. It does what the real one does + * where tests can see it: a span opened with `inSpan` is kept in the async + * context, so a step called inside it is recorded under the span path, and a + * step's origin is its own `"~origin"` option or the innermost span's. Where + * the real SDK stamps a step's `opts.span` and `opts.origin` onto the step + * the executor reports, the stub records them by step ID for `runFunction` to + * read back. It installs nothing when the SDK has the span API itself. + * + * @module + */ + +import { group, Middleware } from "inngest"; +import type { AsyncContext } from "inngest/experimental"; +import { runWithAsyncCtx } from "inngest/experimental"; +import type { SpanInfo } from "../pipeline/spans.ts"; +import { hasSpanApi } from "../pipeline/spans.ts"; + +/** What the stub knows about a step: its spans, outermost first, and origin. */ +export interface StepStamp { + span: SpanInfo[]; + origin?: string; +} + +/** Where a span stack lives in an async context. Contexts copied by spread keep it. */ +const spanStackKey = Symbol("ci.spanStub.spans"); + +interface SpannedContext extends AsyncContext { + [spanStackKey]?: SpanInfo[]; +} + +const stamps = new Map(); + +let installed = false; + +/** + * The current async context, read synchronously like the real span API does. + * The SDK keeps its async storage on a global, and only offers an async getter, + * which would delay every span by a tick and reorder concurrent work. + */ +const currentContext = (): SpannedContext | undefined => { + const cache = ( + globalThis as Record< + symbol, + { resolved?: { getStore: () => AsyncContext | undefined } } | undefined + > + )[Symbol.for("inngest:als")]; + + return cache?.resolved?.getStore(); +}; + +const spanOf = (span: SpanInfo, fn: () => R): R => { + const ctx = currentContext(); + + if (!ctx) { + return fn(); + } + + const spans = [...(ctx[spanStackKey] ?? []), span]; + + const spanned: SpannedContext = { ...ctx, [spanStackKey]: spans }; + + return runWithAsyncCtx(spanned, fn); +}; + +/** The SDK's own `"~span"`, while `removeSpanStub` has it taken off. */ +let hidden: PropertyDescriptor | undefined; + +/** Give the SDK's `group` a `"~span"`, unless it already has one. */ +export const installSpanStub = (): void => { + if (hidden) { + Object.defineProperty(group, "~span", hidden); + + hidden = undefined; + + return; + } + + if (hasSpanApi()) { + return; + } + + Object.defineProperty(group, "~span", { + value: spanOf, + configurable: true, + writable: true, + }); + + installed = true; +}; + +/** + * Take the stub off again, so the SDK looks as it does without the span API. + * An SDK that has the span API itself has it taken off too, until + * `installSpanStub` puts it back. + */ +export const removeSpanStub = (): void => { + if (installed) { + Reflect.deleteProperty(group, "~span"); + + installed = false; + + return; + } + + hidden ??= Object.getOwnPropertyDescriptor(group, "~span"); + + Reflect.deleteProperty(group, "~span"); +}; + +/** Whether the stub is what's providing the span API. */ +export const spanStubInstalled = (): boolean => { + return installed; +}; + +/** What the stub recorded for a step, by the ID CI gave it. */ +export const stampOf = (stepId: string): StepStamp | undefined => { + return stamps.get(stepId); +}; + +/** Record the step `args` is about to run, from the span stack in effect now. */ +const record = (args: unknown[]): void => { + const first = args[0]; + + const options = + typeof first === "string" + ? { id: first } + : (first as { id?: unknown; "~origin"?: unknown } | undefined); + + if (typeof options?.id !== "string") { + return; + } + + const span = currentContext()?.[spanStackKey] ?? []; + + const inherited = [...span].reverse().find((entry) => { + return entry.origin; + })?.origin; + + const own = (options as { "~origin"?: unknown })["~origin"]; + + stamps.set(options.id, { + span, + origin: typeof own === "string" ? own : inherited, + }); +}; + +/** + * Wrap step tools so every call is recorded. The SDK's middleware hooks don't + * hand a step's unknown options to middleware, so the calls are watched where + * they're made. + */ +const recording = (tools: T): T => { + return new Proxy(tools, { + get(target, prop, receiver) { + const value = Reflect.get(target, prop, receiver) as unknown; + + if (typeof value === "function") { + return (...args: unknown[]) => { + record(args); + + return (value as (...a: unknown[]) => unknown).apply(target, args); + }; + } + + if (value && typeof value === "object") { + return recording(value); + } + + return value; + }, + }); +}; + +/** Records each step's span path and origin as it's called. */ +export class SpanStubMiddleware extends Middleware.BaseMiddleware { + readonly id = "ci-span-stub"; + + override transformFunctionInput( + arg: Middleware.TransformFunctionInputArgs, + ): Middleware.TransformFunctionInputArgs { + return { ...arg, ctx: { ...arg.ctx, step: recording(arg.ctx.step) } }; + } +} diff --git a/packages/ci/src/types.ts b/packages/ci/src/types.ts index 07ce386f5..a09e84dd0 100644 --- a/packages/ci/src/types.ts +++ b/packages/ci/src/types.ts @@ -202,7 +202,7 @@ export interface CacheConfig { * ``` */ export interface JobConfig { - /** Unique within the CI client. It's the job's path in the trace and its check name. */ + /** Unique within the CI client. It's the job's check name, and its step IDs start with it. */ id: string; /** * The job to start from: this job runs on a copy of that job's machine, so @@ -232,6 +232,8 @@ export interface JobConfig { * choose between them in the pipeline. */ from?: From; + /** What the trace calls the job. Defaults to `id`. */ + name?: string; /** Machine settings for this job, overriding the pipeline's and the client's. */ machine?: MachineConfig; /** diff --git a/packages/ci/src/units.test.ts b/packages/ci/src/units.test.ts index 23dcfa85c..c15e89fe0 100644 --- a/packages/ci/src/units.test.ts +++ b/packages/ci/src/units.test.ts @@ -710,6 +710,7 @@ describe("looking a cached snapshot up by name", () => { get: () => Promise = async () => { return null; }, + exclude?: string, ) => { const scope = { path: "setup", @@ -728,6 +729,7 @@ describe("looking a cached snapshot up by name", () => { scope, { key: "v1", scope: "global" }, { ownKey: "k1", name }, + exclude, ); }; @@ -793,6 +795,12 @@ describe("looking a cached snapshot up by name", () => { createdAt: snapshot({}).createdAt, }); }); + + test("a snapshot found to be bad is never found again", async () => { + expect(await lookup(listing(snapshot({})), undefined, "s1")).toBe( + undefined, + ); + }); }); describe("snapshot names", () => { diff --git a/packages/ci/vitest.config.ts b/packages/ci/vitest.config.ts index 2a6163f50..d3b49e53c 100644 --- a/packages/ci/vitest.config.ts +++ b/packages/ci/vitest.config.ts @@ -4,6 +4,7 @@ export default defineConfig({ test: { environment: "node", silent: "passed-only", + setupFiles: ["src/testing/setup.ts"], typecheck: { enabled: true, include: ["src/**/*.test.ts"], From ea84b11c3d1d540aa58991f8d6a209b5edc840be Mon Sep 17 00:00:00 2001 From: x Date: Sat, 10 Oct 2026 00:41:01 +0000 Subject: [PATCH 02/18] refactor(ci): span catalog, staged machine start, span kinds folded in Every span is an entry in the `spans` catalog with a required kind, step IDs are built by `joinId`, and starting a machine is a flat staged function in `machine/start.ts`. Folds the CI half of ci2-span-kinds: commands, sandboxes, saves and attempts carry kinds, each statement on a background process is a row of its own, and a snapshot's span is passed to an SDK that takes it (feature-detected by version) and opened by CI otherwise. Names the checkout steps. Adds a `ciTest` harness. Co-Authored-By: Claude Opus 5.5 --- .changeset/ci-trace-spans.md | 2 +- packages/ci/src/cache/cache.ts | 8 +- packages/ci/src/checkout/checkout.ts | 27 +- packages/ci/src/machine/command.ts | 108 +++-- packages/ci/src/machine/from.ts | 4 +- packages/ci/src/machine/machine.ts | 459 ++---------------- packages/ci/src/machine/sandbox.ts | 8 +- packages/ci/src/machine/sandboxErrors.ts | 67 +++ packages/ci/src/machine/spans.test.ts | 379 ++++++--------- packages/ci/src/machine/start.ts | 335 +++++++++++++ .../ci/src/pipeline/cacheGuarantees.test.ts | 2 +- packages/ci/src/pipeline/job.ts | 7 +- packages/ci/src/pipeline/names.ts | 72 ++- packages/ci/src/pipeline/pipeline.test.ts | 7 +- packages/ci/src/pipeline/scope.ts | 20 +- packages/ci/src/pipeline/spans.test.ts | 169 +++++-- packages/ci/src/pipeline/spans.ts | 61 ++- packages/ci/src/testing/README.md | 1 + packages/ci/src/testing/harness.ts | 115 +++++ 19 files changed, 1058 insertions(+), 793 deletions(-) create mode 100644 packages/ci/src/machine/sandboxErrors.ts create mode 100644 packages/ci/src/machine/start.ts create mode 100644 packages/ci/src/testing/harness.ts diff --git a/.changeset/ci-trace-spans.md b/.changeset/ci-trace-spans.md index d3ef514f5..abe6dd96e 100644 --- a/.changeset/ci-trace-spans.md +++ b/.changeset/ci-trace-spans.md @@ -2,4 +2,4 @@ "@inngest/ci": minor --- -Group a run's steps into trace spans (one per job, one per command, one per extra machine), name CI's own steps for what they do, and mark them as CI's work so a trace can tell them from yours. Give a job a `name` to change what its span is called. +Group a run's steps into trace spans (one per job, one per command, one per extra machine), name CI's own steps for what they do, and mark them as CI's work so a trace can tell them from yours. Give a job a `name` to change what its span is called. Every span has a kind (job, sandbox, command, snapshot, github or attempt), and each statement on a background process (`output`, `exited`, `kill`) is a row of its own. Saving a sandbox is one `snapshot` row on an `inngest` that takes the span, and the span is opened by CI on older ones. diff --git a/packages/ci/src/cache/cache.ts b/packages/ci/src/cache/cache.ts index d53d856b0..173b327c5 100644 --- a/packages/ci/src/cache/cache.ts +++ b/packages/ci/src/cache/cache.ts @@ -13,7 +13,7 @@ import { tagStep } from "../pipeline/metadata.ts"; import { ciStep, traceName } from "../pipeline/names.ts"; import type { CiJobScope, CiRunScope } from "../pipeline/scope.ts"; -import { countApi, rootRunIdOf, scopeSeparator } from "../pipeline/scope.ts"; +import { countApi, joinId, rootRunIdOf } from "../pipeline/scope.ts"; import type { CacheConfig, CacheKey, @@ -283,7 +283,7 @@ export const cacheTarget = async ( countApi("cache"); const ownKey = (await run.step.run( - ciStep(`${job.path}${scopeSeparator}cache:key`, traceName.checkCache), + ciStep(joinId(job.path, "cache:key"), traceName.checkCache), async () => { await tagStep(run, { kind: "cache", job: job.path }); @@ -462,7 +462,7 @@ export const lookupCache = async ( const { run } = scope; const found = (await run.step.run( - ciStep(`${scope.path}${scopeSeparator}cache:lookup`, traceName.lookUpCache), + ciStep(joinId(scope.path, "cache:lookup"), traceName.lookUpCache), async () => { await tagStep(run, { kind: "cache", job: scope.path }); @@ -516,7 +516,7 @@ export const lookupBeforeBuild = async ( exclude?: string, ): Promise => { const found = (await run.step.run( - ciStep(`${job.stepPath}${scopeSeparator}lookup`, traceName.lookUpCache), + ciStep(joinId(job.stepPath, "lookup"), traceName.lookUpCache), async () => { await tagStep(run, { kind: "cache", job: job.path }); diff --git a/packages/ci/src/checkout/checkout.ts b/packages/ci/src/checkout/checkout.ts index 1914f6a0b..cb8256309 100644 --- a/packages/ci/src/checkout/checkout.ts +++ b/packages/ci/src/checkout/checkout.ts @@ -7,12 +7,13 @@ import { CiUsageError } from "../errors.ts"; import { ensureMachine } from "../machine/machine.ts"; +import { traceName } from "../pipeline/names.ts"; import type { CiRunScope, MachineHandle } from "../pipeline/scope.ts"; import { countApi, defaultCwd, + joinId, requireJobScope, - scopeSeparator, } from "../pipeline/scope.ts"; import type { RepoContext } from "../types.ts"; import { shellEscape } from "../util.ts"; @@ -88,15 +89,21 @@ export const checkout = async (opts: CheckoutOptions = {}): Promise => { } const machine = await ensureMachine(scope); - const stepId = `${scope.path}${scopeSeparator}checkout`; - - await run.step.run({ id: stepId, name: stepId }, async () => { - if (local) { - return uploadWorkingTree(run, machine, local.path, target); - } - - return cloneFromGithub(run, machine, repo as RepoContext, opts, target); - }); + const stepId = joinId(scope.path, "checkout"); + + await run.step.run( + { + id: stepId, + name: local ? traceName.uploadWorkingTree : traceName.cloneRepository, + }, + async () => { + if (local) { + return uploadWorkingTree(run, machine, local.path, target); + } + + return cloneFromGithub(run, machine, repo as RepoContext, opts, target); + }, + ); scope.cwd ??= target; }; diff --git a/packages/ci/src/machine/command.ts b/packages/ci/src/machine/command.ts index 6e0a0f732..7cf420bee 100644 --- a/packages/ci/src/machine/command.ts +++ b/packages/ci/src/machine/command.ts @@ -13,14 +13,14 @@ import { CommandFailedError, CommandTimeoutError, } from "../errors.ts"; -import { ciSpan, ciStep, traceName } from "../pipeline/names.ts"; +import { ciStep, spans, traceName } from "../pipeline/names.ts"; import type { CiJobScope, MachineHandle } from "../pipeline/scope.ts"; import { countApi, defaultCwd, + joinId, nextStepId, requireJobScope, - scopeSeparator, } from "../pipeline/scope.ts"; import { inSpan } from "../pipeline/spans.ts"; import type { @@ -176,7 +176,7 @@ export const buildShellString = ( * command's span is yours, but how CI runs it isn't. */ const subStep = (stepId: string, suffix: string, name: string) => { - return ciStep(`${stepId}${scopeSeparator}${suffix}`, name); + return ciStep(joinId(stepId, suffix), name); }; class CommandBuilder implements Command { @@ -268,7 +268,8 @@ class CommandBuilder implements Command { const stepId = this.stepId(scope); const machine = await ensureMachine(scope); - // Each call back into the process re-enters the command's span. + // Starting, awaiting, reading and killing the process are separate + // statements, so each is a row of its own, never one span re-entered. const process = await this.inSpan(scope, stepId, () => { return startProcess(machine, stepId, this.spawnOptions(scope)); }); @@ -278,39 +279,60 @@ class CommandBuilder implements Command { // Numbers the follow-up steps, so calling `exited()` or `output()` more // than once never reuses a step ID. const nextWait = counter(); + const nextExited = counter(); return { id: process.id, exited: () => { - return this.inSpan(scope, stepId, async () => { - const polled = await pollUntilTerminal({ - scope, - machine, - process, - stepId, - nextWait, - }); - - return readResult({ process: polled.process, stepId, argv, secrets }); - }); + return this.inSpan( + scope, + stepId, + async () => { + const polled = await pollUntilTerminal({ + scope, + machine, + process, + stepId, + nextWait, + }); + + return readResult({ + process: polled.process, + stepId, + argv, + secrets, + }); + }, + { suffix: `exited #${nextExited()}`, label: "exited" }, + ); }, kill: async (signal = 15) => { - await this.inSpan(scope, stepId, () => { - return process.signal( - subStep(stepId, "kill", traceName.stopProcess), - { signal }, - ); - }); + await this.inSpan( + scope, + stepId, + () => { + return process.signal( + subStep(stepId, "kill", traceName.stopProcess), + { signal }, + ); + }, + { suffix: "kill", label: "kill" }, + ); }, output: async (opts) => { - const output = await this.inSpan(scope, stepId, () => { - return process.getOutput( - subStep(stepId, `output #${nextWait()}`, traceName.readOutput), - { - tailBytes: opts?.tailBytes ?? outputTailBytes, - }, - ); - }); + const suffix = `output #${nextWait()}`; + + const output = await this.inSpan( + scope, + stepId, + () => { + return process.getOutput( + subStep(stepId, suffix, traceName.readOutput), + { tailBytes: opts?.tailBytes ?? outputTailBytes }, + ); + }, + { suffix, label: "output" }, + ); const decoded = decodeChunks(output); @@ -371,12 +393,23 @@ class CommandBuilder implements Command { }); } - /** Run `fn` in the command's span, in its machine's span if it's an extra. */ - private inSpan(scope: CiJobScope, stepId: string, fn: () => T): T { - const span = { - id: stepId, - name: traceName.command(this.commandText(), this.state.label), - }; + /** + * Run `fn` in the command's span, in its machine's span if it's an extra. + * A statement on a background process, such as its `kill`, is a span of its + * own beside the command's. + */ + private inSpan( + scope: CiJobScope, + stepId: string, + fn: () => T, + statement?: { suffix: string; label: string }, + ): T { + const span = spans.command( + stepId, + this.commandText(), + this.state.label, + statement, + ); return inMachineSpan(scope, () => { return inSpan(span, fn); @@ -413,10 +446,7 @@ class CommandBuilder implements Command { const result = await this.inSpan(scope, stepId, () => { return attempts === 1 ? runAttempt() - : inSpan( - ciSpan(`attempt-${attempt}`, traceName.attempt(attempt)), - runAttempt, - ); + : inSpan(spans.attempt(attempt), runAttempt); }); scope.run.ci.hooks.commandFinished(scope, attemptInfo, result); diff --git a/packages/ci/src/machine/from.ts b/packages/ci/src/machine/from.ts index 7c1a7ed03..5a1bd1066 100644 --- a/packages/ci/src/machine/from.ts +++ b/packages/ci/src/machine/from.ts @@ -25,9 +25,9 @@ import { adoptBuilt, invokeBuild, validateInput } from "../pipeline/job.ts"; import type { CiJobScope, CiRunScope } from "../pipeline/scope.ts"; import { countApi, + joinId, outsideJobs, rebuildSuffix, - scopeSeparator, } from "../pipeline/scope.ts"; import type { AnyJob, JobConfig, JobRef } from "../types.ts"; import { errorMessage, hash, stableStringify } from "../util.ts"; @@ -486,7 +486,7 @@ const requestRebuild = ({ if (replacing.broken) { const gone = await deleteSnapshot( run, - `${path}${scopeSeparator}cache:delete`, + joinId(path, "cache:delete"), replacing.snapshotId, ); diff --git a/packages/ci/src/machine/machine.ts b/packages/ci/src/machine/machine.ts index a0c4f6779..2a38b7a1e 100644 --- a/packages/ci/src/machine/machine.ts +++ b/packages/ci/src/machine/machine.ts @@ -7,51 +7,32 @@ import type { Inngest } from "inngest"; import type { CachedSnapshot, CacheTarget } from "../cache/cache.ts"; -import { resolveTakenName, snapshotState } from "../cache/cache.ts"; -import { CiUsageError } from "../errors.ts"; -import { ciSpan, ciStep, traceName } from "../pipeline/names.ts"; +import { resolveTakenName } from "../cache/cache.ts"; +import { ciStep, spans, traceName } from "../pipeline/names.ts"; import type { CiJobScope, CiRunScope, MachineHandle, } from "../pipeline/scope.ts"; import { - defaultCwd, inJobSpan, isRestoring, + joinId, runInScope, scopeSeparator, } from "../pipeline/scope.ts"; -import { inSpan } from "../pipeline/spans.ts"; -import type { MachineConfig } from "../types.ts"; import { - boundedName, + inSnapshotSpan, + inSpan, + snapshotSpanOption, +} from "../pipeline/spans.ts"; +import { errorMessage, isSandboxNotFound, isSnapshotNotFound, - slug, } from "../util.ts"; - -/** - * Memory is paired with vCPU count, so a job only picks one number. - */ -const memoryForVcpu = { 1: 1024, 2: 2048, 4: 4096 } as const; - -export const resolveMachineConfig = ( - config: MachineConfig | undefined, -): { vcpu: 1 | 2 | 4; memoryMb: number } => { - const vcpu = config?.vcpu ?? 2; - - return { vcpu, memoryMb: memoryForVcpu[vcpu] }; -}; - -/** - * The name a machine is created with. It carries the run ID so orphaned - * machines can be found and destroyed by the cleanup function. - */ -export const machineName = (runId: string, path: string): string => { - return boundedName(`ci-${runId}-${slug(path)}`); -}; +import { errorStatus, hasCode } from "./sandboxErrors.ts"; +import { startMachine } from "./start.ts"; /** * Create this scope's machine if it doesn't have one yet. @@ -65,14 +46,14 @@ export const ensureMachine = async ( ): Promise => { scope.machine ??= inJobSpan(scope.run, scope.jobPath, () => { return inMachineSpan(scope, () => { - const stepId = `${scope.path}${scopeSeparator}machine`; + const stepId = joinId(scope.path, "machine"); // Everything in it is CI's work, including falling back from a // snapshot that won't start. - const span = ciSpan(stepId, traceName.startMachine(scope.fromJobIds[0])); + const span = spans.startMachine(stepId, scope.fromJobIds[0]); return inSpan(span, () => { - return createMachine(scope, stepId); + return startMachine(scope, stepId); }); }); }); @@ -109,391 +90,8 @@ export const inMachineSpan = (scope: CiJobScope, fn: () => R): R => { return fn(); } - const span = { - id: scope.path, - name: traceName.extraMachine(scope.path, scope.jobPath), - }; - - return inSpan(span, fn); -}; - -/** - * Run once on every machine before its first command. - * - * WORKAROUNDS (Sandboxes API), delete each part once the platform covers it: - * - Commands run in `/work` by default, and a sandbox won't start a process in - * a directory that doesn't exist, so it's created here. `checkout()` also - * makes it, but a job doesn't have to check out. - * - Sandboxes boot with the loopback interface down, so nothing can listen on - * or reach `127.0.0.1`, which breaks services started with `.background()` - * and `waitForHttp`/`waitForPort`. Bringing it up is a no-op once the - * platform does it itself. - */ -export const machineSetupScript = `mkdir -p ${defaultCwd} && (ip link set lo up 2>/dev/null || true)`; - -/** A machine that has started and been set up. */ -interface Started { - // biome-ignore lint/suspicious/noExplicitAny: DurableSandbox - sandbox: any; -} - -const createMachine = async ( - scope: CiJobScope, - /** The create step's ID, which other steps for the machine build on. */ - stepId: string, -): Promise => { - const { run } = scope; - const tools = run.sandboxTools; - - if (!tools) { - throw new CiUsageError( - "This pipeline has no `step.sandbox` tools. `createCi` adds `sandboxMiddleware()` to the functions it creates, so this usually means the function was created another way.", - ); - } - - const name = machineName(run.runId, scope.path); - const create = { id: stepId, name: traceName.createMachine }; - - const machineConfig = resolveMachineConfig( - scope.config.machine ?? run.machine ?? run.ci.defaultMachine, - ); - - /** Create a machine, from a snapshot if one is given, and set it up. */ - const start = async ( - createStep: { id: string; name: string }, - options: { - name: string; - snapshotId?: string; - }, - ): Promise => { - const sandbox = options.snapshotId - ? await tools.create(createStep, { - name: options.name, - snapshotId: options.snapshotId, - }) - : await tools.create(createStep, { - name: options.name, - ...machineConfig, - }); - - await sandbox.commands.run( - { - id: `${createStep.id}${scopeSeparator}setup`, - name: traceName.prepareWorkspace, - }, - ["/bin/sh", "-c", machineSetupScript], - ); - - return { sandbox }; - }; - - const startFresh = (note: string) => { - scope.fromSnapshotId = undefined; - scope.startNote = note; - scope.restoreFallback = scope.rebuildParent; - - run.ci.hooks.activity(run, scope.jobPath, note); - - return start( - { - id: `${stepId}${scopeSeparator}fresh`, - name: traceName.createFreshMachine, - }, - { name: machineName(run.runId, `${scope.path} fresh`) }, - ); - }; - - const parentId = scope.fromJobIds[0] ?? "the parent"; - const snapshotId = scope.fromSnapshotId; - - let started: Started; - - if (snapshotId) { - // Every job starting from a snapshot checks it for itself, so what a job - // does never depends on whether a sibling got there first. Only the - // rebuild of the parent, which each of them awaits, is shared. - run.ci.hooks.activity( - run, - scope.jobPath, - scope.startNote ?? "creating machine…", - ); - - let probed: Started | undefined; - - /** Why the snapshot can't be used, if it can't. */ - let bad: string | undefined; - - /** How to replace it, when it can't be used. */ - let rebuild = { broken: false, unnamed: false }; - - try { - probed = await start(create, { name, snapshotId }); - } catch (error) { - if (!isStartFailure(error)) { - throw error; - } - - // The failed start still holds `name`, so what replaces it gets - // another, and the stuck machine is cleared away meanwhile. - await discardFailedStart(run, stepId, name, error); - - bad = `wouldn't start (${errorMessage(error)})`; - - // WORKAROUND (Sandboxes API): a snapshot can't be told to be broken, so - // it is retried before it is deleted. Delete this once snapshots or - // images report their own health. Today a snapshot has only the - // statuses `CREATING`, `READY`, `DELETING` and `DELETED`, and a broken - // one still says `READY`. A start that times out says no more: the - // machine may have been stuck, or the node busy, as easily as the - // snapshot bad. Deleting on the first one would let a single blip - // destroy a cache that is shared, since a pull request restores its - // base branch's snapshots, and every run after it would build again. So - // only a snapshot that fails to start a second time is broken enough to - // delete. Capacity errors (`compute_unavailable`, 429 and 503) are not - // start failures at all, and fail the job as any other would. - const state = await snapshotState( - run, - `${stepId}${scopeSeparator}snapshot-state`, - snapshotId, - ); - - if (state === "creating") { - // Another build is still taking it and may yet finish, so this - // run rebuilds without contending for its name. - rebuild = { broken: false, unnamed: true }; - } else if (state === "ready") { - const restartStepId = `${stepId}${scopeSeparator}restart`; - const restartName = machineName(run.runId, `${scope.path} restart`); - - try { - // With the default wait, the same as the first try's: a shorter one - // would make a slow node more likely to fail the retry, and a second - // failure deletes the snapshot. - probed = await start( - { id: restartStepId, name: traceName.restartMachine }, - { - name: restartName, - snapshotId, - }, - ); - - bad = undefined; - - run.warnings.push( - `retried: snapshot of \`${parentId}\` needed a retry to start (${errorMessage(error)}), so \`${scope.path}\` started it on the second try`, - ); - } catch (second) { - if (!isStartFailure(second)) { - throw second; - } - - await discardFailedStart(run, restartStepId, restartName, second); - - bad = `wouldn't start twice (${errorMessage(second)})`; - - rebuild = { broken: true, unnamed: false }; - } - } - } - - if (probed && !bad) { - started = probed; - } else { - run.warnings.push( - `fell back: snapshot of \`${parentId}\` ${bad}, so \`${scope.path}\` rebuilt it`, - ); - - const note = `rebuilding ${parentId} · bad snapshot`; - - scope.startNote = note; - - run.ci.hooks.activity(run, scope.jobPath, note); - - // Shared: jobs that had the same trouble wait for one rebuild, which - // also deletes a broken snapshot once. - const replacement = await scope.rebuildSnapshot?.(rebuild); - - if (replacement) { - scope.startNote = `starting ${parentId}`; - - run.ci.hooks.activity(run, scope.jobPath, scope.startNote); - - const retryStepId = `${stepId}${scopeSeparator}retry`; - const retryName = machineName(run.runId, `${scope.path} retry`); - - try { - started = await start( - { id: retryStepId, name: traceName.retryCreate }, - { - name: retryName, - snapshotId: replacement, - }, - ); - } catch (error) { - if (!isStartFailure(error)) { - throw error; - } - - // Restores may still be broken while fresh machines work, which is - // where this job went before there was a replacement to try. - await discardFailedStart(run, retryStepId, retryName, error); - - started = await startFresh(note); - } - } else { - started = await startFresh(note); - } - } - } else { - run.ci.hooks.activity( - run, - scope.jobPath, - scope.startNote ?? "creating machine…", - ); - - started = await start(create, { name }); - } - - const handle: MachineHandle = { - sandbox: started.sandbox, - name, - id: started.sandbox.id, - }; - - return handle; + return inSpan(spans.extraMachine(scope.path, scope.jobPath), fn); }; - -/** - * Best-effort cleanup of a machine that failed to start. The platform keeps - * such a machine (stuck in STARTING) and its name, so it is destroyed here, and - * the end of the run finds it by its name if this couldn't. - * - * Never throws: a machine that can't be cleaned up now must not fail the job. - */ -const discardFailedStart = async ( - run: CiRunScope, - stepId: string, - name: string, - error: unknown, -): Promise => { - const knownId = (error as { cause?: { sandboxId?: string } })?.cause - ?.sandboxId; - - try { - await run.step.run( - { - id: `${stepId}${scopeSeparator}discard`, - name: traceName.discardMachine, - }, - async (): Promise<{ id?: string }> => { - // Errors are swallowed inside the step so it never retries. - let id = knownId; - - try { - let cursor: string | undefined; - - while (!id) { - const page = await run.ci.client.sandboxes.list({ - ...(cursor ? { cursor } : {}), - limit: 100, - }); - - id = page.items.find( - (sandbox: { name: string; status: string }) => { - return sandbox.name === name && sandbox.status !== "TERMINATED"; - }, - )?.id; - - if (id || !page.page.hasMore) { - break; - } - - cursor = page.page.cursor; - } - - if (id) { - const sandbox = await run.ci.client.sandboxes.get(id); - - await sandbox?.destroy(); - } - } catch { - // The run's cleanup finds it again by its name. - } - - return id ? { id } : {}; - }, - ); - } catch { - // Best effort only. - } -}; - -/** - * Whether a machine failed to start, as opposed to a request that was refused. - * Only a snapshot restore falls back on it: a plain machine that won't start - * fails the job with its reason. - */ -const isStartFailure = (error: unknown): boolean => { - const codes = ["sandbox_start_timed_out", "sandbox_start_failed"]; - - // No capacity, or too many requests, says nothing about the snapshot. - if (isRetryable(error)) { - return false; - } - - return ( - codes.some((code) => { - return hasCode(error, code); - }) || /did not reach RUNNING/i.test(errorMessage(error)) - ); -}; - -/** - * Whether the platform says to try again, as it does for no capacity - * (`compute_unavailable`) and rate limits (429 and 503). - */ -const isRetryable = (error: unknown): boolean => { - const seen = error as - | { retryable?: boolean; cause?: { retryable?: boolean } } - | undefined; - const status = errorStatus(error); - - return ( - seen?.retryable === true || - seen?.cause?.retryable === true || - hasCode(error, "compute_unavailable") || - hasCode(error, "rate_limited") || - status === 429 || - status === 503 - ); -}; - -/** Whether a Sandboxes error has a code, on the error or the step error's cause. */ -const hasCode = (error: unknown, code: string): boolean => { - const seen = error as - | { code?: string; cause?: { code?: string } } - | undefined; - - return seen?.code === code || seen?.cause?.code === code; -}; - -/** A Sandboxes error's HTTP status, whether on the error or its cause. */ -const errorStatus = (error: unknown): number | undefined => { - const seen = error as - | { status?: number; cause?: { status?: number } } - | undefined; - - return seen?.status ?? seen?.cause?.status; -}; - -/** Run `fn` in a job's "Save sandbox" span, which holds snapshotting it. */ -const inSaveSpan = (jobPath: string, fn: () => R): R => { - return inSpan( - ciSpan(`${jobPath}${scopeSeparator}save`, traceName.saveMachine), - fn, - ); -}; - /** How a job's snapshot is named. */ export interface SnapshotCache { target: CacheTarget; @@ -523,9 +121,17 @@ export interface TakenSnapshot { reused: boolean; } -/** A snapshot step: its ID is the step's, its name reads as an action. */ -const snapshotStep = (id: string) => { - return { id, name: traceName.snapshotMachine }; +/** + * A snapshot step: its ID is the step's, its name reads as an action. Saving + * a job's sandbox is one statement, so its row is the job's "Save sandbox" + * span, which the SDK is told of where it can take it. + */ +const snapshotStep = (jobPath: string, id: string) => { + return { + id, + name: traceName.snapshotMachine, + ...snapshotSpanOption(spans.save(jobPath)), + }; }; /** @@ -549,7 +155,7 @@ export const snapshotMachine = async ( const handle = await scope.machine; - return inSaveSpan(scope.path, () => { + return inSnapshotSpan(spans.save(scope.path), () => { return takeSnapshot(run, scope.path, handle, cache); }); }; @@ -562,12 +168,12 @@ const takeSnapshot = async ( ): Promise => { run.ci.hooks.activity(run, jobPath, "snapshotting machine…"); - const stepId = `${jobPath}${scopeSeparator}snapshot`; + const stepId = joinId(jobPath, "snapshot"); try { return cache && !cache.unnamed ? await createNamedSnapshot(run, handle, jobPath, stepId, cache) - : await createRunSnapshot(handle, stepId); + : await createRunSnapshot(handle, jobPath, stepId); } catch (error) { if (!isSnapshotUnavailable(error)) { throw error; @@ -601,7 +207,9 @@ const createNamedSnapshot = async ( const name = target.name; const attempt = async (id: string): Promise => { - const snapshot = await handle.sandbox.snapshot(snapshotStep(id), { name }); + const snapshot = await handle.sandbox.snapshot(snapshotStep(jobPath, id), { + name, + }); return { snapshotId: snapshot.id, @@ -659,7 +267,7 @@ const createNamedSnapshot = async ( } const unnamed = await handle.sandbox.snapshot( - snapshotStep(`${stepId} (unnamed)`), + snapshotStep(jobPath, `${stepId} (unnamed)`), ); // The name was refused or couldn't be freed, so the job's snapshot falls @@ -676,9 +284,10 @@ const createNamedSnapshot = async ( */ const createRunSnapshot = async ( handle: MachineHandle, + jobPath: string, stepId: string, ): Promise => { - const snapshot = await handle.sandbox.snapshot(snapshotStep(stepId)); + const snapshot = await handle.sandbox.snapshot(snapshotStep(jobPath, stepId)); return { snapshotId: snapshot.id, reused: false }; }; diff --git a/packages/ci/src/machine/sandbox.ts b/packages/ci/src/machine/sandbox.ts index 616475ebd..f0caad9d5 100644 --- a/packages/ci/src/machine/sandbox.ts +++ b/packages/ci/src/machine/sandbox.ts @@ -6,11 +6,7 @@ import { waitForHttp, waitForPort } from "../checkout/wait.ts"; import type { CiJobScope } from "../pipeline/scope.ts"; -import { - countApi, - requireJobScope, - scopeSeparator, -} from "../pipeline/scope.ts"; +import { countApi, joinId, requireJobScope } from "../pipeline/scope.ts"; import type { Duration, ExtraMachine, MachineConfig } from "../types.ts"; import { createCommandTag } from "./command.ts"; import { ensureMachine } from "./machine.ts"; @@ -53,7 +49,7 @@ export const sandbox = async ( const job = requireJobScope("sandbox"); countApi("sandbox"); - const path = `${job.jobPath}${scopeSeparator}${name}`; + const path = joinId(job.jobPath, name); const existing = job.extras?.get(name); diff --git a/packages/ci/src/machine/sandboxErrors.ts b/packages/ci/src/machine/sandboxErrors.ts new file mode 100644 index 000000000..7b10821f8 --- /dev/null +++ b/packages/ci/src/machine/sandboxErrors.ts @@ -0,0 +1,67 @@ +/** + * What a Sandboxes error says about itself: whether a machine failed to + * start, whether the platform says to try again, and the code and status it + * carries, on the error or on the step error's cause. + * + * @module + */ + +import { errorMessage } from "../util.ts"; + +/** + * Whether a machine failed to start, as opposed to a request that was refused. + * Only a snapshot restore falls back on it: a plain machine that won't start + * fails the job with its reason. + */ +export const isStartFailure = (error: unknown): boolean => { + const codes = ["sandbox_start_timed_out", "sandbox_start_failed"]; + + // No capacity, or too many requests, says nothing about the snapshot. + if (isRetryable(error)) { + return false; + } + + return ( + codes.some((code) => { + return hasCode(error, code); + }) || /did not reach RUNNING/i.test(errorMessage(error)) + ); +}; + +/** + * Whether the platform says to try again, as it does for no capacity + * (`compute_unavailable`) and rate limits (429 and 503). + */ +const isRetryable = (error: unknown): boolean => { + const seen = error as + | { retryable?: boolean; cause?: { retryable?: boolean } } + | undefined; + const status = errorStatus(error); + + return ( + seen?.retryable === true || + seen?.cause?.retryable === true || + hasCode(error, "compute_unavailable") || + hasCode(error, "rate_limited") || + status === 429 || + status === 503 + ); +}; + +/** Whether a Sandboxes error has a code, on the error or the step error's cause. */ +export const hasCode = (error: unknown, code: string): boolean => { + const seen = error as + | { code?: string; cause?: { code?: string } } + | undefined; + + return seen?.code === code || seen?.cause?.code === code; +}; + +/** A Sandboxes error's HTTP status, whether on the error or its cause. */ +export const errorStatus = (error: unknown): number | undefined => { + const seen = error as + | { status?: number; cause?: { status?: number } } + | undefined; + + return seen?.status ?? seen?.cause?.status; +}; diff --git a/packages/ci/src/machine/spans.test.ts b/packages/ci/src/machine/spans.test.ts index 681b0df14..b88bbaf4d 100644 --- a/packages/ci/src/machine/spans.test.ts +++ b/packages/ci/src/machine/spans.test.ts @@ -2,148 +2,54 @@ * Tests of the trace spans CI groups its steps under, and what it names them: * one span per job, and in it one per machine, one per command, and one per * attempt for a command with retries, with GitHub check updates in a span of - * their own. Spans and names never change a step's ID. A failed run of a - * command ends in a failing step, so its span shows the failure. CI's own - * work carries CI's origin, and what you wrote carries none. + * their own. Every span has a kind, and a statement on a background process + * is a row of its own. Spans and names never change a step's ID. A failed run + * of a command ends in a failing step, so its span shows the failure. CI's + * own work carries CI's origin, and what you wrote carries none. * * @module */ -import { version as sdkVersion, step } from "inngest"; +import { step } from "inngest"; import { describe, expect, test } from "vitest"; import { CommandFailedError } from "../errors.ts"; -import { consoleReporter } from "../github/auth.ts"; -import { createCi } from "../pipeline/createCi.ts"; import { ciOrigin } from "../pipeline/names.ts"; -import { createCiTestClient } from "../testing/client.ts"; -import { prEvent } from "../testing/events.ts"; -import type { CommandScript, FakeSandboxApi } from "../testing/fakeSandbox.ts"; import { createFakeSandboxApi } from "../testing/fakeSandbox.ts"; -import { runFunction } from "../testing/runFunction.ts"; +import type { Ci } from "../testing/harness.ts"; +import { ciTest, drawTrace } from "../testing/harness.ts"; import { version } from "../version.ts"; import { $ } from "./command.ts"; import { sandbox } from "./sandbox.ts"; -type Ci = ReturnType; - -interface TraceNode { - label: string; - children: Map; -} - -/** - * Draw a run's trace as an indented tree, the way the UI nests it: each span - * once, where its first step is, with its kind in brackets, and each step - * under its spans. - */ -const drawTrace = ( - result: Awaited>, - /** Suffix each row with who it says did it, as in `Create sandbox <- ci`. */ - withOrigins = false, -): string => { - const root: TraceNode = { label: "", children: new Map() }; - - const by = (origin: string | undefined) => { - if (!withOrigins || !origin) { - return ""; - } - - const names: Record = { - [ciOrigin]: "ci", - [`inngest@${sdkVersion}`]: "inngest", - }; - - return ` <- ${names[origin] ?? origin}`; - }; - - for (const stepId of result.stepIds) { - let node = root; - - for (const span of result.spans[stepId] ?? []) { - const key = `span:${span.id}`; - const label = span.kind ? `${span.name} [${span.kind}]` : span.name; - - const child = node.children.get(key) ?? { - label: `${label}${by(span.origin)}`, - children: new Map(), - }; - - node.children.set(key, child); - - node = child; - } - - node.children.set(`step:${stepId}`, { - label: `${result.names[stepId] ?? stepId}${by(result.origins[stepId])}`, - children: new Map(), - }); - } - - const lines: string[] = []; - - const draw = (node: TraceNode, depth: number) => { - for (const child of node.children.values()) { - lines.push(`${" ".repeat(depth)}${child.label}`); - - draw(child, depth + 1); - } - }; - - draw(root, 0); - - return lines.join("\n"); -}; - -/** Run the pipeline whose handler `define` makes from a fresh client. */ -const runPipeline = async ( - define: (ci: Ci) => () => Promise, - scripts: CommandScript[] = [], - api: FakeSandboxApi = createFakeSandboxApi(), -) => { - api.script(scripts); - - const ci = createCi(createCiTestClient(api), { - github: consoleReporter(), - }); - - const pipeline = ci.pipeline( - { id: "pr", on: [{ event: "github/pull_request.opened" }] }, - define(ci), - ); - - return runFunction(pipeline, { event: prEvent }); -}; - /** Run a job with a captured, a managed, a retried and a background command. */ const run = () => { - return runPipeline( - (ci) => { - return ci.job("test", async () => { - await $`quick`.timeout("30s"); + return ciTest({ + scripts: [ + { match: "slow", ticks: 1, stdout: "done" }, + { match: "flaky", exitCode: 1 }, + { match: "serve", ticks: 1, stdout: "listening" }, + ], + }).run((ci) => { + return ci.job("test", async () => { + await $`quick`.timeout("30s"); - await $`slow`; + await $`slow`; - try { - await $`flaky`.retries(1); - } catch { - // Both attempts fail. - } + try { + await $`flaky`.retries(1); + } catch { + // Both attempts fail. + } - const server = await $`serve`.background(); + const server = await $`serve`.background(); - await server.output(); + await server.output(); - await server.exited(); + await server.exited(); - await server.kill(); - }); - }, - [ - { match: "slow", ticks: 1, stdout: "done" }, - { match: "flaky", exitCode: 1 }, - { match: "serve", ticks: 1, stdout: "listening" }, - ], - ); + await server.kill(); + }); + }); }; /** @@ -173,68 +79,73 @@ const rowsPipeline = (ci: Ci) => { }); }; -/** The span of a command `test` ran. */ -const command = (label: string) => { - return { id: `test › ${label}`, name: `$ ${label}` }; -}; - const origin = ciOrigin; -const github = [{ id: "github", name: "GitHub", origin }]; -const job = { id: "test", name: "test", kind: "job" }; -const machine = [job, { id: "test › machine", name: "Start sandbox", origin }]; -const quick = [job, command("quick")]; -const slow = [job, command("slow")]; -const attempt1 = [ - job, - command("flaky"), - { id: "attempt-1", name: "Attempt 1", origin }, -]; -const attempt2 = [ +const span = (id: string, name: string, kind: string) => { + return { id, name, kind }; +}; +const github = [{ ...span("github", "GitHub", "github"), origin }]; +const job = span("test", "test", "job"); +const machine = [ job, - command("flaky"), - { id: "attempt-2", name: "Attempt 2", origin }, + { ...span("test › machine", "Start sandbox", "sandbox"), origin }, ]; -const serve = [job, command("serve")]; +const command = (label: string) => { + return [job, span(`test › ${label}`, `$ ${label}`, "command")]; +}; +const attempt = (n: number) => { + return [ + ...command("flaky"), + { ...span(`attempt-${n}`, `Attempt ${n}`, "attempt"), origin }, + ]; +}; + +/** A statement on the background `serve`, a row of its own beside the command's. */ +const statement = (suffix: string, label: string) => { + return [ + job, + span(`test › serve › ${suffix}`, `$ serve (${label})`, "command"), + ]; +}; describe("spans", () => { - test("group each job, machine, command and retried attempt", async () => { + test("group each job, machine, command and retried attempt, each with its kind", async () => { const result = await run(); expect(result.type).toBe("function-resolved"); - // Background calls re-enter their command's span. Check updates are in - // the run's GitHub span, and cleanup is in none. + // Each statement on a background process is a span of its own. Check + // updates are in the run's GitHub span, and cleanup is in none. expect(result.spans).toEqual({ "github › check:pr:start": github, "github › check:test:start": github, "test › machine": machine, "test › machine › setup": machine, - "test › quick": quick, - "test › slow › start": slow, - "test › slow › wait #1": slow, - "test › slow › check #1": slow, - "test › slow › wait #2": slow, - "test › slow › check #2": slow, - "test › slow › output": slow, - "test › flaky #attempt-1 › start": attempt1, - "test › flaky #attempt-1 › wait #1": attempt1, - "test › flaky #attempt-1 › check #1": attempt1, - "test › flaky #attempt-1 › output": attempt1, - "test › flaky #attempt-1 › exit": attempt1, + "test › quick": command("quick"), + "test › slow › start": command("slow"), + "test › slow › wait #1": command("slow"), + "test › slow › check #1": command("slow"), + "test › slow › wait #2": command("slow"), + "test › slow › check #2": command("slow"), + "test › slow › output": command("slow"), + "test › flaky #attempt-1 › start": attempt(1), + "test › flaky #attempt-1 › wait #1": attempt(1), + "test › flaky #attempt-1 › check #1": attempt(1), + "test › flaky #attempt-1 › output": attempt(1), + "test › flaky #attempt-1 › exit": attempt(1), "github › check:test:attempt:1": github, - "test › flaky #attempt-2 › start": attempt2, - "test › flaky #attempt-2 › wait #1": attempt2, - "test › flaky #attempt-2 › check #1": attempt2, - "test › flaky #attempt-2 › output": attempt2, - "test › flaky #attempt-2 › exit": attempt2, - "test › serve › start": serve, - "test › serve › output #1": serve, - "test › serve › wait #2": serve, - "test › serve › check #2": serve, - "test › serve › wait #3": serve, - "test › serve › check #3": serve, - "test › serve › output": serve, - "test › serve › kill": serve, + "test › flaky #attempt-2 › start": attempt(2), + "test › flaky #attempt-2 › wait #1": attempt(2), + "test › flaky #attempt-2 › check #1": attempt(2), + "test › flaky #attempt-2 › output": attempt(2), + "test › flaky #attempt-2 › exit": attempt(2), + "test › serve › start": command("serve"), + "test › serve › output #1": statement("output #1", "output"), + "test › serve › wait #2": statement("exited #1", "exited"), + "test › serve › check #2": statement("exited #1", "exited"), + "test › serve › wait #3": statement("exited #1", "exited"), + "test › serve › check #3": statement("exited #1", "exited"), + "test › serve › output": statement("exited #1", "exited"), + "test › serve › kill": statement("kill", "kill"), "github › check:test:complete": github, "github › check:jobs:complete": github, "github › check:pr:complete": github, @@ -316,7 +227,7 @@ describe("spans", () => { expect(attempt1Steps.at(-1)).toBe("test › flaky #attempt-1 › exit"); - expect(result.spans["test › flaky #attempt-1 › exit"]).toEqual(attempt1); + expect(result.spans["test › flaky #attempt-1 › exit"]).toEqual(attempt(1)); expect(result.names["test › flaky #attempt-1 › exit"]).toBe( "Exited with code 1", @@ -329,55 +240,41 @@ describe("spans", () => { }); test("record nothing more for a passing or `.nothrow()` command", async () => { - const result = await runPipeline( - (ci) => { - return ci.job("test", async () => { - await $`pass`; - - await $`fail`.nothrow(); - }); - }, - [{ match: "fail", exitCode: 1 }], - ); + const result = await ciTest({ + scripts: [{ match: "fail", exitCode: 1 }], + }).run((ci) => { + return ci.job("test", async () => { + await $`pass`; + + await $`fail`.nothrow(); + }); + }); expect(result.type).toBe("function-resolved"); - expect(result.stepIds).toEqual([ - "github › check:pr:start", - "github › check:test:start", - "test › machine", - "test › machine › setup", - "test › pass › start", - "test › pass › wait #1", - "test › pass › check #1", - "test › pass › output", - "test › fail › start", - "test › fail › wait #1", - "test › fail › check #1", - "test › fail › output", - "github › check:test:complete", - "github › check:jobs:complete", - "github › check:pr:complete", - "pipeline › cleanup", - "pipeline › cleanup:snapshots", - ]); + expect( + result.stepIds.filter((id) => { + return id.endsWith("› exit"); + }), + ).toEqual([]); }); test("still retry after a failed run, and throw the same error at the end", async () => { let caught: unknown; - const result = await runPipeline( - (ci) => { - return ci.job("test", async () => { - try { - await $`flaky`.retries(1); - } catch (error) { - caught = error; - } - }); - }, - [{ match: "flaky", exitCode: 1, stdout: "out", stderr: "flake" }], - ); + const result = await ciTest({ + scripts: [ + { match: "flaky", exitCode: 1, stdout: "out", stderr: "flake" }, + ], + }).run((ci) => { + return ci.job("test", async () => { + try { + await $`flaky`.retries(1); + } catch (error) { + caught = error; + } + }); + }); expect(result.type).toBe("function-resolved"); @@ -396,7 +293,7 @@ describe("spans", () => { }); test("keep the build of a parent outside every job's span", async () => { - const result = await runPipeline((ci) => { + const result = await ciTest().run((ci) => { const base = ci.job("base", async () => { await $`install`; }); @@ -435,13 +332,13 @@ describe("spans", () => { }); test("name a pipeline's rows for what they are", async () => { - const result = await runPipeline(rowsPipeline); + const result = await ciTest().run(rowsPipeline); expect(result.type).toBe("function-resolved"); expect(drawTrace(result)).toBe( [ - "GitHub", + "GitHub [github]", " Create check: pr", " Report test: started", " Report test: passed", @@ -450,25 +347,25 @@ describe("spans", () => { "Look up cache", "Build base in its own run", "test [job]", - " Start sandbox from base", + " Start sandbox from base [sandbox]", " Create sandbox", " Prepare workspace", - " $ pnpm test", + " $ pnpm test [command]", " Start process", " Wait 1s", " Poll process", " Read output", - " lint", + " lint [command]", " Start process", " Wait 1s", " Poll process", " Read output", " my-step", - " api", - " Start sandbox", + " api [sandbox]", + " Start sandbox [sandbox]", " Create sandbox", " Prepare workspace", - " $ pnpm start", + " $ pnpm start [command]", " Run and read output", "Clean up sandboxes", "Clean up snapshots", @@ -477,18 +374,16 @@ describe("spans", () => { }); test("mark the work CI does for you, and nothing you wrote", async () => { - const result = await runPipeline(rowsPipeline); + const result = await ciTest().run(rowsPipeline); expect(result.type).toBe("function-resolved"); expect(ciOrigin).toBe(`@inngest/ci@${version}`); - // Jobs, commands, `my-step` and the `api` sandbox are yours. `snapshot()` - // opens its span without an origin, so it inherits the one of the CI span - // it runs in, while its own steps are marked by the SDK. + // Jobs, commands, `my-step` and the `api` sandbox are yours. expect(drawTrace(result, true)).toBe( [ - "GitHub <- ci", + "GitHub [github] <- ci", " Create check: pr <- ci", " Report test: started <- ci", " Report test: passed <- ci", @@ -497,25 +392,25 @@ describe("spans", () => { "Look up cache <- ci", "Build base in its own run <- ci", "test [job]", - " Start sandbox from base <- ci", + " Start sandbox from base [sandbox] <- ci", " Create sandbox <- ci", " Prepare workspace <- ci", - " $ pnpm test", + " $ pnpm test [command]", " Start process <- ci", " Wait 1s <- ci", " Poll process <- ci", " Read output <- ci", - " lint", + " lint [command]", " Start process <- ci", " Wait 1s <- ci", " Poll process <- ci", " Read output <- ci", " my-step", - " api", - " Start sandbox <- ci", + " api [sandbox]", + " Start sandbox [sandbox] <- ci", " Create sandbox <- ci", " Prepare workspace <- ci", - " $ pnpm start", + " $ pnpm start [command]", " Run and read output <- ci", "Clean up sandboxes <- ci", "Clean up snapshots <- ci", @@ -550,19 +445,15 @@ describe("spans", () => { return fetch(input, init); }; - const result = await runPipeline( - (ci) => { - const parent = ci.job("base", async () => { - await $`pnpm install`; - }); - - return ci.job({ id: "test", from: parent }, async () => { - await $`pnpm test`; - }); - }, - [], - api, - ); + const result = await ciTest({ api }).run((ci) => { + const parent = ci.job("base", async () => { + await $`pnpm install`; + }); + + return ci.job({ id: "test", from: parent }, async () => { + await $`pnpm test`; + }); + }); expect(result.type).toBe("function-resolved"); diff --git a/packages/ci/src/machine/start.ts b/packages/ci/src/machine/start.ts new file mode 100644 index 000000000..5ade4f307 --- /dev/null +++ b/packages/ci/src/machine/start.ts @@ -0,0 +1,335 @@ +/** + * Starting a job's machine: creating it, setting it up, and falling back when + * the snapshot it restores won't start. + * + * @module + */ + +import { snapshotState } from "../cache/cache.ts"; +import { CiUsageError } from "../errors.ts"; +import { traceName } from "../pipeline/names.ts"; +import type { + CiJobScope, + CiRunScope, + MachineHandle, +} from "../pipeline/scope.ts"; +import { defaultCwd, joinId } from "../pipeline/scope.ts"; +import type { MachineConfig } from "../types.ts"; +import { boundedName, errorMessage, slug } from "../util.ts"; +import { isStartFailure } from "./sandboxErrors.ts"; + +/** + * Memory is paired with vCPU count, so a job only picks one number. + */ +const memoryForVcpu = { 1: 1024, 2: 2048, 4: 4096 } as const; + +const resolveMachineConfig = ( + config: MachineConfig | undefined, +): { vcpu: 1 | 2 | 4; memoryMb: number } => { + const vcpu = config?.vcpu ?? 2; + + return { vcpu, memoryMb: memoryForVcpu[vcpu] }; +}; + +/** + * The name a machine is created with. It carries the run ID so orphaned + * machines can be found and destroyed by the cleanup function. + */ +const machineName = (runId: string, path: string): string => { + return boundedName(`ci-${runId}-${slug(path)}`); +}; + +/** + * Run once on every machine before its first command. + * + * WORKAROUNDS (Sandboxes API), delete each part once the platform covers it: + * - Commands run in `/work` by default, and a sandbox won't start a process in + * a directory that doesn't exist, so it's created here. `checkout()` also + * makes it, but a job doesn't have to check out. + * - Sandboxes boot with the loopback interface down, so nothing can listen on + * or reach `127.0.0.1`, which breaks services started with `.background()` + * and `waitForHttp`/`waitForPort`. Bringing it up is a no-op once the + * platform does it itself. + */ +export const machineSetupScript = `mkdir -p ${defaultCwd} && (ip link set lo up 2>/dev/null || true)`; + +/** + * The stages a machine is started in. Only the first is tried every time: + * the rest follow a snapshot that won't start. Each has its own step ID, + * machine name and trace name, all derived from the key. + */ +const stages = { + first: traceName.createMachine, + restart: traceName.restartMachine, + retry: traceName.retryCreate, + fresh: traceName.createFreshMachine, +}; + +type Stage = keyof typeof stages; + +type Sandbox = MachineHandle["sandbox"]; + +/** + * Create this scope's machine, from the snapshot it starts from if it has one. + * + * A snapshot that won't start is retried once, and a second failure makes it + * broken. Either way the job doesn't fail: the parent is rebuilt for it to + * start from, and a fresh machine is the last resort. + */ +export const startMachine = async ( + scope: CiJobScope, + /** The first stage's step ID, which the other stages build on. */ + stepId: string, +): Promise => { + const sandbox = await startSandbox(scope, stepId); + + return { + sandbox, + name: machineName(scope.run.runId, scope.path), + id: sandbox.id, + }; +}; + +const startSandbox = async ( + scope: CiJobScope, + stepId: string, +): Promise => { + const { run } = scope; + const tools = run.sandboxTools; + + if (!tools) { + throw new CiUsageError( + "This pipeline has no `step.sandbox` tools. `createCi` adds `sandboxMiddleware()` to the functions it creates, so this usually means the function was created another way.", + ); + } + + const machineConfig = resolveMachineConfig( + scope.config.machine ?? run.machine ?? run.ci.defaultMachine, + ); + + const idOf = (stage: Stage): string => { + return stage === "first" ? stepId : joinId(stepId, stage); + }; + + const nameOf = (stage: Stage): string => { + return machineName( + run.runId, + stage === "first" ? scope.path : `${scope.path} ${stage}`, + ); + }; + + /** Create a machine at a stage, from a snapshot if one is given, and set it up. */ + const start = async (stage: Stage, snapshotId?: string): Promise => { + const step = { id: idOf(stage), name: stages[stage] }; + const name = nameOf(stage); + + const sandbox = await tools.create( + step, + snapshotId ? { name, snapshotId } : { name, ...machineConfig }, + ); + + await sandbox.commands.run( + { id: joinId(step.id, "setup"), name: traceName.prepareWorkspace }, + ["/bin/sh", "-c", machineSetupScript], + ); + + return sandbox; + }; + + /** + * Start from a snapshot, or the start failure that says it won't. The failed + * start still holds its name, so what follows gets another, and the stuck + * machine is cleared away meanwhile. + */ + const restore = async ( + stage: Stage, + snapshotId: string, + ): Promise<{ started: Sandbox } | { failure: unknown }> => { + try { + return { started: await start(stage, snapshotId) }; + } catch (failure) { + if (!isStartFailure(failure)) { + throw failure; + } + + await discardFailedStart(run, idOf(stage), nameOf(stage), failure); + + return { failure }; + } + }; + + const startFresh = (note: string): Promise => { + scope.fromSnapshotId = undefined; + scope.startNote = note; + scope.restoreFallback = scope.rebuildParent; + + run.ci.hooks.activity(run, scope.jobPath, note); + + return start("fresh"); + }; + + run.ci.hooks.activity( + run, + scope.jobPath, + scope.startNote ?? "creating machine…", + ); + + const snapshotId = scope.fromSnapshotId; + + if (!snapshotId) { + return start("first"); + } + + const parentId = scope.fromJobIds[0] ?? "the parent"; + + // Every job starting from a snapshot checks it for itself, so what a job + // does never depends on whether a sibling got there first. Only the + // rebuild of the parent, which each of them awaits, is shared. + const first = await restore("first", snapshotId); + + if ("started" in first) { + return first.started; + } + + /** Why the snapshot can't be used. */ + let bad = `wouldn't start (${errorMessage(first.failure)})`; + + /** How to replace it. */ + let rebuild = { broken: false, unnamed: false }; + + // WORKAROUND (Sandboxes API): a snapshot can't be told to be broken, so + // it is retried before it is deleted. Delete this once snapshots or + // images report their own health. Today a snapshot has only the + // statuses `CREATING`, `READY`, `DELETING` and `DELETED`, and a broken + // one still says `READY`. A start that times out says no more: the + // machine may have been stuck, or the node busy, as easily as the + // snapshot bad. Deleting on the first one would let a single blip + // destroy a cache that is shared, since a pull request restores its + // base branch's snapshots, and every run after it would build again. So + // only a snapshot that fails to start a second time is broken enough to + // delete. Capacity errors (`compute_unavailable`, 429 and 503) are not + // start failures at all, and fail the job as any other would. + const state = await snapshotState( + run, + joinId(stepId, "snapshot-state"), + snapshotId, + ); + + if (state === "creating") { + // Another build is still taking it and may yet finish, so this + // run rebuilds without contending for its name. + rebuild = { broken: false, unnamed: true }; + } else if (state === "ready") { + // With the default wait, the same as the first try's: a shorter one + // would make a slow node more likely to fail the retry, and a second + // failure deletes the snapshot. + const second = await restore("restart", snapshotId); + + if ("started" in second) { + run.warnings.push( + `retried: snapshot of \`${parentId}\` needed a retry to start (${errorMessage(first.failure)}), so \`${scope.path}\` started it on the second try`, + ); + + return second.started; + } + + bad = `wouldn't start twice (${errorMessage(second.failure)})`; + + rebuild = { broken: true, unnamed: false }; + } + + run.warnings.push( + `fell back: snapshot of \`${parentId}\` ${bad}, so \`${scope.path}\` rebuilt it`, + ); + + const note = `rebuilding ${parentId} · bad snapshot`; + + scope.startNote = note; + + run.ci.hooks.activity(run, scope.jobPath, note); + + // Shared: jobs that had the same trouble wait for one rebuild, which + // also deletes a broken snapshot once. + const replacement = await scope.rebuildSnapshot?.(rebuild); + + if (replacement) { + scope.startNote = `starting ${parentId}`; + + run.ci.hooks.activity(run, scope.jobPath, scope.startNote); + + // Restores may still be broken while fresh machines work, which is where + // this job went before there was a replacement to try. + const retried = await restore("retry", replacement); + + if ("started" in retried) { + return retried.started; + } + } + + return startFresh(note); +}; + +/** + * Best-effort cleanup of a machine that failed to start. The platform keeps + * such a machine (stuck in STARTING) and its name, so it is destroyed here, and + * the end of the run finds it by its name if this couldn't. + * + * Never throws: a machine that can't be cleaned up now must not fail the job. + */ +const discardFailedStart = async ( + run: CiRunScope, + stepId: string, + name: string, + error: unknown, +): Promise => { + const knownId = (error as { cause?: { sandboxId?: string } })?.cause + ?.sandboxId; + + try { + await run.step.run( + { + id: joinId(stepId, "discard"), + name: traceName.discardMachine, + }, + async (): Promise<{ id?: string }> => { + // Errors are swallowed inside the step so it never retries. + let id = knownId; + + try { + let cursor: string | undefined; + + while (!id) { + const page = await run.ci.client.sandboxes.list({ + ...(cursor ? { cursor } : {}), + limit: 100, + }); + + id = page.items.find( + (sandbox: { name: string; status: string }) => { + return sandbox.name === name && sandbox.status !== "TERMINATED"; + }, + )?.id; + + if (id || !page.page.hasMore) { + break; + } + + cursor = page.page.cursor; + } + + if (id) { + const sandbox = await run.ci.client.sandboxes.get(id); + + await sandbox?.destroy(); + } + } catch { + // The run's cleanup finds it again by its name. + } + + return id ? { id } : {}; + }, + ); + } catch { + // Best effort only. + } +}; diff --git a/packages/ci/src/pipeline/cacheGuarantees.test.ts b/packages/ci/src/pipeline/cacheGuarantees.test.ts index c34aa0b88..8814edda2 100644 --- a/packages/ci/src/pipeline/cacheGuarantees.test.ts +++ b/packages/ci/src/pipeline/cacheGuarantees.test.ts @@ -18,7 +18,7 @@ import { describe, expect, test } from "vitest"; import { consoleReporter } from "../github/auth.ts"; import { $ } from "../machine/command.ts"; -import { machineSetupScript } from "../machine/machine.ts"; +import { machineSetupScript } from "../machine/start.ts"; import { createCiTestClient } from "../testing/client.ts"; import { prEvent, prTrigger } from "../testing/events.ts"; import { createFakeSandboxApi } from "../testing/fakeSandbox.ts"; diff --git a/packages/ci/src/pipeline/job.ts b/packages/ci/src/pipeline/job.ts index 447587df6..e482b6d8d 100644 --- a/packages/ci/src/pipeline/job.ts +++ b/packages/ci/src/pipeline/job.ts @@ -38,10 +38,10 @@ import type { CiJobScope, CiRunScope } from "./scope.ts"; import { getRunScope, inJobSpan, + joinId, matrixOriginOf, rootRunIdOf, runJobBody, - scopeSeparator, } from "./scope.ts"; export interface RegisteredJob { @@ -234,10 +234,7 @@ export const invokeBuild = async ({ try { output = (await run.step.invoke( - ciStep( - `${stepPath}${scopeSeparator}build`, - traceName.buildInOwnRun(path), - ), + ciStep(joinId(stepPath, "build"), traceName.buildInOwnRun(path)), { function: run.ci.cacheBuild(), data }, )) as CacheBuildResult | null; } catch (error) { diff --git a/packages/ci/src/pipeline/names.ts b/packages/ci/src/pipeline/names.ts index 82964bf51..475a259b5 100644 --- a/packages/ci/src/pipeline/names.ts +++ b/packages/ci/src/pipeline/names.ts @@ -14,7 +14,8 @@ import type { CheckConclusion } from "../types.ts"; import { formatDuration } from "../util.ts"; import { version } from "../version.ts"; import type { CiRunScope } from "./scope.ts"; -import { scopeSeparator } from "./scope.ts"; +import { joinId, scopeSeparator } from "./scope.ts"; +import type { SpanInfo, SpanKind } from "./spans.ts"; import { originOption } from "./spans.ts"; /** @@ -25,9 +26,72 @@ import { originOption } from "./spans.ts"; */ export const ciOrigin = `@inngest/ci@${version}`; +/** A span for something you wrote or asked for, such as a job or a command. */ +const userSpan = (kind: SpanKind, id: string, name: string): SpanInfo => { + return { kind, id, name }; +}; + /** A span CI opens for its own work, such as starting a sandbox. */ -export const ciSpan = (id: string, name: string) => { - return { id, name, origin: ciOrigin }; +const ciSpan = (kind: SpanKind, id: string, name: string): SpanInfo => { + return { ...userSpan(kind, id, name), origin: ciOrigin }; +}; + +/** + * Every span CI opens: its kind, ID, name and whose work it is, in one place. + * A new span is an entry here and a `inSpan(spans.x(...), fn)` where it + * opens. Spans of what you wrote have no origin. + */ +export const spans = { + /** A job, which holds everything it does. */ + job: (run: CiRunScope, path: string): SpanInfo => { + return userSpan("job", path, traceName.job(run, path)); + }, + + /** The run's one span for reporting to GitHub. */ + github: (): SpanInfo => { + return ciSpan("github", "github", traceName.github); + }, + + /** A machine starting, with its fallbacks. */ + startMachine: (stepId: string, parent?: string): SpanInfo => { + return ciSpan("sandbox", stepId, traceName.startMachine(parent)); + }, + + /** An extra sandbox, named as `sandbox()` was. */ + extraMachine: (path: string, jobPath: string): SpanInfo => { + return userSpan("sandbox", path, traceName.extraMachine(path, jobPath)); + }, + + /** + * A command. A statement on a background process, such as its `kill`, is a + * span of its own beside the command's. + */ + command: ( + stepId: string, + text: string, + label?: string, + statement?: { suffix: string; label: string }, + ): SpanInfo => { + const name = traceName.command(text, label); + + return statement + ? userSpan( + "command", + joinId(stepId, statement.suffix), + `${name} (${statement.label})`, + ) + : userSpan("command", stepId, name); + }, + + /** One attempt of a command that has retries. */ + attempt: (attempt: number): SpanInfo => { + return ciSpan("attempt", `attempt-${attempt}`, traceName.attempt(attempt)); + }, + + /** Saving a job's sandbox: one statement, whatever steps it takes. */ + save: (jobPath: string): SpanInfo => { + return ciSpan("snapshot", joinId(jobPath, "save"), traceName.saveMachine); + }, }; /** @@ -150,6 +214,8 @@ export const traceName = { deleteBadSnapshot: "Delete bad snapshot", checkSnapshotState: "Check snapshot state", + cloneRepository: "Clone repository", + uploadWorkingTree: "Upload working tree", findChangedFiles: "Find changed files", recordStartTime: "Record start time", diff --git a/packages/ci/src/pipeline/pipeline.test.ts b/packages/ci/src/pipeline/pipeline.test.ts index 27d0b17b6..caa4f65fa 100644 --- a/packages/ci/src/pipeline/pipeline.test.ts +++ b/packages/ci/src/pipeline/pipeline.test.ts @@ -22,12 +22,9 @@ import { import { consoleReporter, githubToken } from "../github/auth.ts"; import { github } from "../github/index.ts"; import { $ } from "../machine/command.ts"; -import { - destroyOrphans, - destroyRunMachines, - machineSetupScript, -} from "../machine/machine.ts"; +import { destroyOrphans, destroyRunMachines } from "../machine/machine.ts"; import { sandbox } from "../machine/sandbox.ts"; +import { machineSetupScript } from "../machine/start.ts"; import { report } from "../report.ts"; import { createCiTestClient } from "../testing/client.ts"; import { prEvent, prTrigger } from "../testing/events.ts"; diff --git a/packages/ci/src/pipeline/scope.ts b/packages/ci/src/pipeline/scope.ts index 653f1c2ba..b4f7932b4 100644 --- a/packages/ci/src/pipeline/scope.ts +++ b/packages/ci/src/pipeline/scope.ts @@ -19,7 +19,7 @@ import type { } from "../types.ts"; import type { CacheBuildData, CacheBuildResult } from "./cacheBuild.ts"; import type { CiHooks } from "./hooks.ts"; -import { ciSpan, traceName } from "./names.ts"; +import { spans } from "./names.ts"; import { inSpan } from "./spans.ts"; /** @@ -29,6 +29,11 @@ import { inSpan } from "./spans.ts"; */ export const scopeSeparator = " › "; +/** A step ID built from parts, as in `test › machine › setup`. */ +export const joinId = (...parts: string[]): string => { + return parts.join(scopeSeparator); +}; + /** What a rebuilt job's path adds to its ID, as in `base (rebuild)`. */ export const rebuildSuffix = " (rebuild)"; @@ -416,7 +421,7 @@ export const nextStepId = ( scopePath: string | undefined, label: string, ): string => { - const base = scopePath ? `${scopePath}${scopeSeparator}${label}` : label; + const base = scopePath ? joinId(scopePath, label) : label; const seen = (run.counters.get(base) ?? 0) + 1; run.counters.set(base, seen); @@ -519,7 +524,7 @@ export const withScopePreserved = (tools: T): T => { const prefixStepId = (idOrOptions: unknown, prefix: string): unknown => { if (typeof idOrOptions === "string") { return { - id: `${prefix}${scopeSeparator}${idOrOptions}`, + id: joinId(prefix, idOrOptions), name: idOrOptions, }; } @@ -534,7 +539,7 @@ const prefixStepId = (idOrOptions: unknown, prefix: string): unknown => { return { ...opts, - id: `${prefix}${scopeSeparator}${opts.id}`, + id: joinId(prefix, opts.id), name: opts.name ?? opts.id, }; } @@ -553,10 +558,7 @@ export const inJobSpan = ( fn: () => R, ): R => { return runWithAsyncCtx(run.asyncCtx, () => { - return inSpan( - { id: jobPath, name: traceName.job(run, jobPath), kind: "job" }, - fn, - ); + return inSpan(spans.job(run, jobPath), fn); }); }; @@ -577,7 +579,7 @@ export const outsideJobs = (run: CiRunScope, fn: () => R): R => { */ export const inGitHubSpan = (run: CiRunScope, fn: () => R): R => { return runWithAsyncCtx(run.asyncCtx, () => { - return inSpan(ciSpan("github", traceName.github), fn); + return inSpan(spans.github(), fn); }); }; diff --git a/packages/ci/src/pipeline/spans.test.ts b/packages/ci/src/pipeline/spans.test.ts index 47fd17a5d..100f1de8c 100644 --- a/packages/ci/src/pipeline/spans.test.ts +++ b/packages/ci/src/pipeline/spans.test.ts @@ -1,46 +1,46 @@ /** * Tests of the span helpers, and of CI on an SDK without the span API: with * none, functions still run, step IDs and names are exactly what they are - * with it, and the trace just has no spans or origins. The rest of the suite - * runs with the test stub in place. + * with it, and the trace just has no spans or origins. The same goes for an + * SDK that doesn't take a snapshot's own span. The rest of the suite runs + * with the test stub in place. * * @module */ import { group, step } from "inngest"; -import { afterEach, beforeEach, describe, expect, test } from "vitest"; -import { consoleReporter } from "../github/auth.ts"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { $ } from "../machine/command.ts"; -import { createCiTestClient } from "../testing/client.ts"; -import { prEvent } from "../testing/events.ts"; -import { createFakeSandboxApi } from "../testing/fakeSandbox.ts"; -import { runFunction } from "../testing/runFunction.ts"; +import { snapshotMachine } from "../machine/machine.ts"; +import { ciTest } from "../testing/harness.ts"; import { installSpanStub, removeSpanStub } from "../testing/spanStub.ts"; -import { createCi } from "./createCi.ts"; -import { hasSpanApi, inSpan, originOption } from "./spans.ts"; +import { ciOrigin } from "./names.ts"; +import type { CiJobScope } from "./scope.ts"; +import { + hasSpanApi, + inSnapshotSpan, + inSpan, + originOption, + sdk, + snapshotSpanOption, + versionAtLeast, +} from "./spans.ts"; /** Run a job with a command and a step of your own. */ const run = () => { - const ci = createCi(createCiTestClient(createFakeSandboxApi()), { - github: consoleReporter(), - }); - - const job: () => Promise = ci.job("test", async () => { - await $`pnpm test`; + return ciTest().run((ci) => { + return ci.job("test", async () => { + await $`pnpm test`; - await step.run("mine", () => { - return "done"; + await step.run("mine", () => { + return "done"; + }); }); }); - - const pipeline = ci.pipeline( - { id: "pr", on: [{ event: "github/pull_request.opened" }] }, - job, - ); - - return runFunction(pipeline, { event: prEvent }); }; +const span = { id: "a", name: "A", kind: "job" } as const; + describe("without the SDK's span API", () => { beforeEach(() => { removeSpanStub(); @@ -57,8 +57,6 @@ describe("without the SDK's span API", () => { }); test("inSpan runs the function and returns what it returns", async () => { - const span = { id: "a", name: "A" }; - expect(inSpan(span, () => 7)).toBe(7); await expect( @@ -109,8 +107,8 @@ describe("with the SDK's span API", () => { const calls: unknown[] = []; Object.defineProperty(group, "~span", { - value: (span: unknown, fn: () => unknown) => { - calls.push(span); + value: (opened: unknown, fn: () => unknown) => { + calls.push(opened); return `wrapped ${String(fn())}`; }, @@ -118,13 +116,7 @@ describe("with the SDK's span API", () => { }); try { - const span = { id: "a", name: "A", kind: "job" }; - - expect( - inSpan(span, () => { - return "inner"; - }), - ).toBe("wrapped inner"); + expect(inSpan(span, () => "inner")).toBe("wrapped inner"); expect(calls).toEqual([span]); } finally { @@ -144,3 +136,108 @@ describe("originOption", () => { }); }); }); + +describe("versionAtLeast", () => { + test.each([ + ["4.23.1", true], + ["4.23.2", true], + ["4.24.0", true], + ["5.0.0", true], + ["4.23.1-pr.1", true], + ["4.23.0", false], + ["4.22.9", false], + ["3.99.99", false], + ])("%s", (version, expected) => { + expect(versionAtLeast(version, "4.23.1")).toBe(expected); + }); +}); + +describe("a snapshot's span", () => { + /** What a job's snapshot passes the SDK, and the spans opened around it. */ + const snapshotWith = async (takesSpan: boolean) => { + vi.spyOn(sdk, "takesSnapshotSpan").mockReturnValue(takesSpan); + + const steps: unknown[] = []; + const opened: unknown[] = []; + + const open = vi.fn((spanned: unknown, fn: () => unknown) => { + opened.push(spanned); + + return fn(); + }); + + Object.defineProperty(group, "~span", { + value: open, + configurable: true, + }); + + const scope = { + path: "test", + machine: Promise.resolve({ + sandbox: { + snapshot: (options: unknown) => { + steps.push(options); + + return Promise.resolve({ id: "snap" }); + }, + }, + }), + run: { ci: { hooks: { activity: vi.fn() } } }, + } as unknown as CiJobScope; + + try { + await snapshotMachine(scope); + } finally { + Reflect.deleteProperty(group, "~span"); + + installSpanStub(); + + vi.restoreAllMocks(); + } + + return { steps, opened }; + }; + + const save = { + id: "test › save", + name: "Save sandbox", + kind: "snapshot", + origin: ciOrigin, + }; + + test("goes to an SDK that takes it, as the step's own span", async () => { + removeSpanStub(); + + const { steps, opened } = await snapshotWith(true); + + expect(steps).toEqual([ + expect.objectContaining({ id: "test › snapshot", "~span": save }), + ]); + + expect(opened).toEqual([]); + }); + + test("is opened here for an SDK that doesn't, which would open its own", async () => { + removeSpanStub(); + + const { steps, opened } = await snapshotWith(false); + + expect(steps).toEqual([expect.not.objectContaining({ "~span": save })]); + + expect(opened).toEqual([save]); + }); + + test("is a step option only where the SDK takes it", () => { + vi.spyOn(sdk, "takesSnapshotSpan").mockReturnValue(true); + + expect(snapshotSpanOption(span)).toEqual({ "~span": span }); + + vi.spyOn(sdk, "takesSnapshotSpan").mockReturnValue(false); + + expect(snapshotSpanOption(span)).toEqual({}); + + expect(inSnapshotSpan(span, () => "ran")).toBe("ran"); + + vi.restoreAllMocks(); + }); +}); diff --git a/packages/ci/src/pipeline/spans.ts b/packages/ci/src/pipeline/spans.ts index 59872b69f..b2471c28e 100644 --- a/packages/ci/src/pipeline/spans.ts +++ b/packages/ci/src/pipeline/spans.ts @@ -10,7 +10,16 @@ * @module */ -import { group } from "inngest"; +import { group, version as sdkVersion } from "inngest"; + +/** What a span stands for. Every span CI opens has one. */ +export type SpanKind = + | "job" + | "sandbox" + | "command" + | "snapshot" + | "github" + | "attempt"; /** A trace span, as the SDK's span API takes it. */ export interface SpanInfo { @@ -18,8 +27,8 @@ export interface SpanInfo { id: string; /** What the span is called in the trace. */ name: string; - /** What kind of span it is, as in `job`. */ - kind?: string; + /** What kind of span it is. */ + kind: SpanKind; /** Who did the work in it, inherited by the steps and spans inside. */ origin?: string; } @@ -68,3 +77,49 @@ export const originOption = ( ): { "~origin": string } => { return { "~origin": origin }; }; + +/** The first SDK whose snapshot takes a caller's `"~span"` as its own span. */ +const snapshotSpanSince = "4.23.1"; + +/** Whether `version` is `since` or later, as in `4.23.1`. */ +export const versionAtLeast = (version: string, since: string): boolean => { + const have = version.split(/[.-]/).map(Number); + + for (const [index, want] of since.split(".").map(Number).entries()) { + const got = have[index] ?? 0; + + if (got !== want) { + return got > want; + } + } + + return true; +}; + +/** + * What the SDK in use can do beyond the span API. Older SDKs ignore the step + * options they don't know, so each is detected here rather than assumed. + */ +export const sdk = { + /** + * Whether a snapshot step's `"~span"` option stands in for the span the SDK + * would open around its create and wait. Older SDKs ignore the option and + * open their own, which then sits inside the span CI opens. + */ + takesSnapshotSpan: (): boolean => { + return versionAtLeast(sdkVersion, snapshotSpanSince); + }, +}; + +/** + * The step option that makes a snapshot one statement in `span`, where the + * SDK takes one. Elsewhere there is none, and `inSnapshotSpan` opens it. + */ +export const snapshotSpanOption = (span: SpanInfo): { "~span"?: SpanInfo } => { + return sdk.takesSnapshotSpan() ? { "~span": span } : {}; +}; + +/** Run `fn`, which takes snapshots, in `span` unless the SDK does it itself. */ +export const inSnapshotSpan = (span: SpanInfo, fn: () => R): R => { + return sdk.takesSnapshotSpan() ? fn() : inSpan(span, fn); +}; diff --git a/packages/ci/src/testing/README.md b/packages/ci/src/testing/README.md index 7893f48ba..bcbca1697 100644 --- a/packages/ci/src/testing/README.md +++ b/packages/ci/src/testing/README.md @@ -7,5 +7,6 @@ Test-only helpers. Not exported and not built. - `client.ts`: an Inngest client wired to the fake sandbox API, which also records the functions it creates. - `events.ts`: the pull request event and trigger tests share. - `spanStub.ts`, `setup.ts`: a stand-in for the SDK's span API, installed before every test file when the SDK has none, so span and origin assertions hold on any SDK. +- `harness.ts`: `ciTest`, a client on the fake API with a `run` for a pipeline a test defines, and `drawTrace` for the trace it left. - `runFunction.ts`: drives a function to completion like the executor, including `step.invoke` of the functions the test client created. Invokes run beside the function, and the function is called again as each one ends; runs of one `event.data` concurrency key take turns. - `schema.ts`: a fake Standard Schema that can also write itself as JSON Schema. diff --git a/packages/ci/src/testing/harness.ts b/packages/ci/src/testing/harness.ts new file mode 100644 index 000000000..4fa5142c2 --- /dev/null +++ b/packages/ci/src/testing/harness.ts @@ -0,0 +1,115 @@ +/** + * What a test of a pipeline starts from: a client on the fake Sandboxes API + * and a `ci` that reports to the console, with `run` driving one pipeline + * whose handler a test defines. `drawTrace` draws the trace such a run left. + * + * @module + */ + +import { version as sdkVersion } from "inngest"; +import { consoleReporter } from "../github/auth.ts"; +import { createCi } from "../pipeline/createCi.ts"; +import { ciOrigin } from "../pipeline/names.ts"; +import { createCiTestClient } from "./client.ts"; +import { prEvent, prTrigger } from "./events.ts"; +import type { CommandScript, FakeSandboxApi } from "./fakeSandbox.ts"; +import { createFakeSandboxApi } from "./fakeSandbox.ts"; +import { runFunction } from "./runFunction.ts"; + +export type Ci = ReturnType; + +export type RunResult = Awaited>; + +export const ciTest = ( + options: { + /** The fake API to run on. Defaults to a fresh one. */ + api?: FakeSandboxApi; + /** What the commands it runs do. */ + scripts?: CommandScript[]; + } = {}, +) => { + const api = options.api ?? createFakeSandboxApi(); + + api.script(options.scripts ?? []); + + const ci = createCi(createCiTestClient(api), { github: consoleReporter() }); + + return { + api, + ci, + /** Run the pipeline whose handler `define` makes from `ci`. */ + run: (define: (ci: Ci) => () => Promise): Promise => { + const pipeline = ci.pipeline({ id: "pr", on: prTrigger }, define(ci)); + + return runFunction(pipeline, { event: prEvent }); + }, + }; +}; + +interface TraceNode { + label: string; + children: Map; +} + +/** + * Draw a run's trace as an indented tree, the way the UI nests it: each span + * once, where its first step is, with its kind in brackets, and each step + * under its spans. + */ +export const drawTrace = ( + result: RunResult, + /** Suffix each row with who it says did it, as in `Create sandbox <- ci`. */ + withOrigins = false, +): string => { + const root: TraceNode = { label: "", children: new Map() }; + + const by = (origin: string | undefined) => { + if (!withOrigins || !origin) { + return ""; + } + + const names: Record = { + [ciOrigin]: "ci", + [`inngest@${sdkVersion}`]: "inngest", + }; + + return ` <- ${names[origin] ?? origin}`; + }; + + for (const stepId of result.stepIds) { + let node = root; + + for (const span of result.spans[stepId] ?? []) { + const key = `span:${span.id}`; + const label = span.kind ? `${span.name} [${span.kind}]` : span.name; + + const child = node.children.get(key) ?? { + label: `${label}${by(span.origin)}`, + children: new Map(), + }; + + node.children.set(key, child); + + node = child; + } + + node.children.set(`step:${stepId}`, { + label: `${result.names[stepId] ?? stepId}${by(result.origins[stepId])}`, + children: new Map(), + }); + } + + const lines: string[] = []; + + const draw = (node: TraceNode, depth: number) => { + for (const child of node.children.values()) { + lines.push(`${" ".repeat(depth)}${child.label}`); + + draw(child, depth + 1); + } + }; + + draw(root, 0); + + return lines.join("\n"); +}; From 5c9fd49cb15b08569ab75eaeba41009e080d1086 Mon Sep 17 00:00:00 2001 From: x Date: Sat, 10 Oct 2026 00:42:13 +0000 Subject: [PATCH 03/18] refactor(ci): keep the staged machine start in machine.ts Moving it to its own file cost more diff than it saved. Co-Authored-By: Claude Opus 5.5 --- packages/ci/src/machine/machine.ts | 383 +++++++++++++++++- packages/ci/src/machine/sandboxErrors.ts | 67 --- packages/ci/src/machine/start.ts | 335 --------------- .../ci/src/pipeline/cacheGuarantees.test.ts | 2 +- packages/ci/src/pipeline/pipeline.test.ts | 7 +- 5 files changed, 386 insertions(+), 408 deletions(-) delete mode 100644 packages/ci/src/machine/sandboxErrors.ts delete mode 100644 packages/ci/src/machine/start.ts diff --git a/packages/ci/src/machine/machine.ts b/packages/ci/src/machine/machine.ts index 2a38b7a1e..8d9751a38 100644 --- a/packages/ci/src/machine/machine.ts +++ b/packages/ci/src/machine/machine.ts @@ -7,7 +7,8 @@ import type { Inngest } from "inngest"; import type { CachedSnapshot, CacheTarget } from "../cache/cache.ts"; -import { resolveTakenName } from "../cache/cache.ts"; +import { resolveTakenName, snapshotState } from "../cache/cache.ts"; +import { CiUsageError } from "../errors.ts"; import { ciStep, spans, traceName } from "../pipeline/names.ts"; import type { CiJobScope, @@ -15,6 +16,7 @@ import type { MachineHandle, } from "../pipeline/scope.ts"; import { + defaultCwd, inJobSpan, isRestoring, joinId, @@ -26,13 +28,14 @@ import { inSpan, snapshotSpanOption, } from "../pipeline/spans.ts"; +import type { MachineConfig } from "../types.ts"; import { + boundedName, errorMessage, isSandboxNotFound, isSnapshotNotFound, + slug, } from "../util.ts"; -import { errorStatus, hasCode } from "./sandboxErrors.ts"; -import { startMachine } from "./start.ts"; /** * Create this scope's machine if it doesn't have one yet. @@ -92,6 +95,380 @@ export const inMachineSpan = (scope: CiJobScope, fn: () => R): R => { return inSpan(spans.extraMachine(scope.path, scope.jobPath), fn); }; +/** + * Memory is paired with vCPU count, so a job only picks one number. + */ +const memoryForVcpu = { 1: 1024, 2: 2048, 4: 4096 } as const; + +const resolveMachineConfig = ( + config: MachineConfig | undefined, +): { vcpu: 1 | 2 | 4; memoryMb: number } => { + const vcpu = config?.vcpu ?? 2; + + return { vcpu, memoryMb: memoryForVcpu[vcpu] }; +}; + +/** + * The name a machine is created with. It carries the run ID so orphaned + * machines can be found and destroyed by the cleanup function. + */ +const machineName = (runId: string, path: string): string => { + return boundedName(`ci-${runId}-${slug(path)}`); +}; + +/** + * Run once on every machine before its first command. + * + * WORKAROUNDS (Sandboxes API), delete each part once the platform covers it: + * - Commands run in `/work` by default, and a sandbox won't start a process in + * a directory that doesn't exist, so it's created here. `checkout()` also + * makes it, but a job doesn't have to check out. + * - Sandboxes boot with the loopback interface down, so nothing can listen on + * or reach `127.0.0.1`, which breaks services started with `.background()` + * and `waitForHttp`/`waitForPort`. Bringing it up is a no-op once the + * platform does it itself. + */ +export const machineSetupScript = `mkdir -p ${defaultCwd} && (ip link set lo up 2>/dev/null || true)`; + +/** + * The stages a machine is started in. Only the first is tried every time: + * the rest follow a snapshot that won't start. Each has its own step ID, + * machine name and trace name, all derived from the key. + */ +const stages = { + first: traceName.createMachine, + restart: traceName.restartMachine, + retry: traceName.retryCreate, + fresh: traceName.createFreshMachine, +}; + +type Stage = keyof typeof stages; + +type Sandbox = MachineHandle["sandbox"]; + +/** + * Create this scope's machine, from the snapshot it starts from if it has one. + * + * A snapshot that won't start is retried once, and a second failure makes it + * broken. Either way the job doesn't fail: the parent is rebuilt for it to + * start from, and a fresh machine is the last resort. + */ +export const startMachine = async ( + scope: CiJobScope, + /** The first stage's step ID, which the other stages build on. */ + stepId: string, +): Promise => { + const sandbox = await startSandbox(scope, stepId); + + return { + sandbox, + name: machineName(scope.run.runId, scope.path), + id: sandbox.id, + }; +}; + +const startSandbox = async ( + scope: CiJobScope, + stepId: string, +): Promise => { + const { run } = scope; + const tools = run.sandboxTools; + + if (!tools) { + throw new CiUsageError( + "This pipeline has no `step.sandbox` tools. `createCi` adds `sandboxMiddleware()` to the functions it creates, so this usually means the function was created another way.", + ); + } + + const machineConfig = resolveMachineConfig( + scope.config.machine ?? run.machine ?? run.ci.defaultMachine, + ); + + const idOf = (stage: Stage): string => { + return stage === "first" ? stepId : joinId(stepId, stage); + }; + + const nameOf = (stage: Stage): string => { + return machineName( + run.runId, + stage === "first" ? scope.path : `${scope.path} ${stage}`, + ); + }; + + /** Create a machine at a stage, from a snapshot if one is given, and set it up. */ + const start = async (stage: Stage, snapshotId?: string): Promise => { + const step = { id: idOf(stage), name: stages[stage] }; + const name = nameOf(stage); + + const sandbox = await tools.create( + step, + snapshotId ? { name, snapshotId } : { name, ...machineConfig }, + ); + + await sandbox.commands.run( + { id: joinId(step.id, "setup"), name: traceName.prepareWorkspace }, + ["/bin/sh", "-c", machineSetupScript], + ); + + return sandbox; + }; + + /** + * Start from a snapshot, or the start failure that says it won't. The failed + * start still holds its name, so what follows gets another, and the stuck + * machine is cleared away meanwhile. + */ + const restore = async ( + stage: Stage, + snapshotId: string, + ): Promise<{ started: Sandbox } | { failure: unknown }> => { + try { + return { started: await start(stage, snapshotId) }; + } catch (failure) { + if (!isStartFailure(failure)) { + throw failure; + } + + await discardFailedStart(run, idOf(stage), nameOf(stage), failure); + + return { failure }; + } + }; + + const startFresh = (note: string): Promise => { + scope.fromSnapshotId = undefined; + scope.startNote = note; + scope.restoreFallback = scope.rebuildParent; + + run.ci.hooks.activity(run, scope.jobPath, note); + + return start("fresh"); + }; + + run.ci.hooks.activity( + run, + scope.jobPath, + scope.startNote ?? "creating machine…", + ); + + const snapshotId = scope.fromSnapshotId; + + if (!snapshotId) { + return start("first"); + } + + const parentId = scope.fromJobIds[0] ?? "the parent"; + + // Every job starting from a snapshot checks it for itself, so what a job + // does never depends on whether a sibling got there first. Only the + // rebuild of the parent, which each of them awaits, is shared. + const first = await restore("first", snapshotId); + + if ("started" in first) { + return first.started; + } + + /** Why the snapshot can't be used. */ + let bad = `wouldn't start (${errorMessage(first.failure)})`; + + /** How to replace it. */ + let rebuild = { broken: false, unnamed: false }; + + // WORKAROUND (Sandboxes API): a snapshot can't be told to be broken, so + // it is retried before it is deleted. Delete this once snapshots or + // images report their own health. Today a snapshot has only the + // statuses `CREATING`, `READY`, `DELETING` and `DELETED`, and a broken + // one still says `READY`. A start that times out says no more: the + // machine may have been stuck, or the node busy, as easily as the + // snapshot bad. Deleting on the first one would let a single blip + // destroy a cache that is shared, since a pull request restores its + // base branch's snapshots, and every run after it would build again. So + // only a snapshot that fails to start a second time is broken enough to + // delete. Capacity errors (`compute_unavailable`, 429 and 503) are not + // start failures at all, and fail the job as any other would. + const state = await snapshotState( + run, + joinId(stepId, "snapshot-state"), + snapshotId, + ); + + if (state === "creating") { + // Another build is still taking it and may yet finish, so this + // run rebuilds without contending for its name. + rebuild = { broken: false, unnamed: true }; + } else if (state === "ready") { + // With the default wait, the same as the first try's: a shorter one + // would make a slow node more likely to fail the retry, and a second + // failure deletes the snapshot. + const second = await restore("restart", snapshotId); + + if ("started" in second) { + run.warnings.push( + `retried: snapshot of \`${parentId}\` needed a retry to start (${errorMessage(first.failure)}), so \`${scope.path}\` started it on the second try`, + ); + + return second.started; + } + + bad = `wouldn't start twice (${errorMessage(second.failure)})`; + + rebuild = { broken: true, unnamed: false }; + } + + run.warnings.push( + `fell back: snapshot of \`${parentId}\` ${bad}, so \`${scope.path}\` rebuilt it`, + ); + + const note = `rebuilding ${parentId} · bad snapshot`; + + scope.startNote = note; + + run.ci.hooks.activity(run, scope.jobPath, note); + + // Shared: jobs that had the same trouble wait for one rebuild, which + // also deletes a broken snapshot once. + const replacement = await scope.rebuildSnapshot?.(rebuild); + + if (replacement) { + scope.startNote = `starting ${parentId}`; + + run.ci.hooks.activity(run, scope.jobPath, scope.startNote); + + // Restores may still be broken while fresh machines work, which is where + // this job went before there was a replacement to try. + const retried = await restore("retry", replacement); + + if ("started" in retried) { + return retried.started; + } + } + + return startFresh(note); +}; + +/** + * Best-effort cleanup of a machine that failed to start. The platform keeps + * such a machine (stuck in STARTING) and its name, so it is destroyed here, and + * the end of the run finds it by its name if this couldn't. + * + * Never throws: a machine that can't be cleaned up now must not fail the job. + */ +const discardFailedStart = async ( + run: CiRunScope, + stepId: string, + name: string, + error: unknown, +): Promise => { + const knownId = (error as { cause?: { sandboxId?: string } })?.cause + ?.sandboxId; + + try { + await run.step.run( + { + id: joinId(stepId, "discard"), + name: traceName.discardMachine, + }, + async (): Promise<{ id?: string }> => { + // Errors are swallowed inside the step so it never retries. + let id = knownId; + + try { + let cursor: string | undefined; + + while (!id) { + const page = await run.ci.client.sandboxes.list({ + ...(cursor ? { cursor } : {}), + limit: 100, + }); + + id = page.items.find( + (sandbox: { name: string; status: string }) => { + return sandbox.name === name && sandbox.status !== "TERMINATED"; + }, + )?.id; + + if (id || !page.page.hasMore) { + break; + } + + cursor = page.page.cursor; + } + + if (id) { + const sandbox = await run.ci.client.sandboxes.get(id); + + await sandbox?.destroy(); + } + } catch { + // The run's cleanup finds it again by its name. + } + + return id ? { id } : {}; + }, + ); + } catch { + // Best effort only. + } +}; + +/** + * Whether a machine failed to start, as opposed to a request that was refused. + * Only a snapshot restore falls back on it: a plain machine that won't start + * fails the job with its reason. + */ +export const isStartFailure = (error: unknown): boolean => { + const codes = ["sandbox_start_timed_out", "sandbox_start_failed"]; + + // No capacity, or too many requests, says nothing about the snapshot. + if (isRetryable(error)) { + return false; + } + + return ( + codes.some((code) => { + return hasCode(error, code); + }) || /did not reach RUNNING/i.test(errorMessage(error)) + ); +}; + +/** + * Whether the platform says to try again, as it does for no capacity + * (`compute_unavailable`) and rate limits (429 and 503). + */ +const isRetryable = (error: unknown): boolean => { + const seen = error as + | { retryable?: boolean; cause?: { retryable?: boolean } } + | undefined; + const status = errorStatus(error); + + return ( + seen?.retryable === true || + seen?.cause?.retryable === true || + hasCode(error, "compute_unavailable") || + hasCode(error, "rate_limited") || + status === 429 || + status === 503 + ); +}; + +/** Whether a Sandboxes error has a code, on the error or the step error's cause. */ +export const hasCode = (error: unknown, code: string): boolean => { + const seen = error as + | { code?: string; cause?: { code?: string } } + | undefined; + + return seen?.code === code || seen?.cause?.code === code; +}; + +/** A Sandboxes error's HTTP status, whether on the error or its cause. */ +export const errorStatus = (error: unknown): number | undefined => { + const seen = error as + | { status?: number; cause?: { status?: number } } + | undefined; + + return seen?.status ?? seen?.cause?.status; +}; + /** How a job's snapshot is named. */ export interface SnapshotCache { target: CacheTarget; diff --git a/packages/ci/src/machine/sandboxErrors.ts b/packages/ci/src/machine/sandboxErrors.ts deleted file mode 100644 index 7b10821f8..000000000 --- a/packages/ci/src/machine/sandboxErrors.ts +++ /dev/null @@ -1,67 +0,0 @@ -/** - * What a Sandboxes error says about itself: whether a machine failed to - * start, whether the platform says to try again, and the code and status it - * carries, on the error or on the step error's cause. - * - * @module - */ - -import { errorMessage } from "../util.ts"; - -/** - * Whether a machine failed to start, as opposed to a request that was refused. - * Only a snapshot restore falls back on it: a plain machine that won't start - * fails the job with its reason. - */ -export const isStartFailure = (error: unknown): boolean => { - const codes = ["sandbox_start_timed_out", "sandbox_start_failed"]; - - // No capacity, or too many requests, says nothing about the snapshot. - if (isRetryable(error)) { - return false; - } - - return ( - codes.some((code) => { - return hasCode(error, code); - }) || /did not reach RUNNING/i.test(errorMessage(error)) - ); -}; - -/** - * Whether the platform says to try again, as it does for no capacity - * (`compute_unavailable`) and rate limits (429 and 503). - */ -const isRetryable = (error: unknown): boolean => { - const seen = error as - | { retryable?: boolean; cause?: { retryable?: boolean } } - | undefined; - const status = errorStatus(error); - - return ( - seen?.retryable === true || - seen?.cause?.retryable === true || - hasCode(error, "compute_unavailable") || - hasCode(error, "rate_limited") || - status === 429 || - status === 503 - ); -}; - -/** Whether a Sandboxes error has a code, on the error or the step error's cause. */ -export const hasCode = (error: unknown, code: string): boolean => { - const seen = error as - | { code?: string; cause?: { code?: string } } - | undefined; - - return seen?.code === code || seen?.cause?.code === code; -}; - -/** A Sandboxes error's HTTP status, whether on the error or its cause. */ -export const errorStatus = (error: unknown): number | undefined => { - const seen = error as - | { status?: number; cause?: { status?: number } } - | undefined; - - return seen?.status ?? seen?.cause?.status; -}; diff --git a/packages/ci/src/machine/start.ts b/packages/ci/src/machine/start.ts deleted file mode 100644 index 5ade4f307..000000000 --- a/packages/ci/src/machine/start.ts +++ /dev/null @@ -1,335 +0,0 @@ -/** - * Starting a job's machine: creating it, setting it up, and falling back when - * the snapshot it restores won't start. - * - * @module - */ - -import { snapshotState } from "../cache/cache.ts"; -import { CiUsageError } from "../errors.ts"; -import { traceName } from "../pipeline/names.ts"; -import type { - CiJobScope, - CiRunScope, - MachineHandle, -} from "../pipeline/scope.ts"; -import { defaultCwd, joinId } from "../pipeline/scope.ts"; -import type { MachineConfig } from "../types.ts"; -import { boundedName, errorMessage, slug } from "../util.ts"; -import { isStartFailure } from "./sandboxErrors.ts"; - -/** - * Memory is paired with vCPU count, so a job only picks one number. - */ -const memoryForVcpu = { 1: 1024, 2: 2048, 4: 4096 } as const; - -const resolveMachineConfig = ( - config: MachineConfig | undefined, -): { vcpu: 1 | 2 | 4; memoryMb: number } => { - const vcpu = config?.vcpu ?? 2; - - return { vcpu, memoryMb: memoryForVcpu[vcpu] }; -}; - -/** - * The name a machine is created with. It carries the run ID so orphaned - * machines can be found and destroyed by the cleanup function. - */ -const machineName = (runId: string, path: string): string => { - return boundedName(`ci-${runId}-${slug(path)}`); -}; - -/** - * Run once on every machine before its first command. - * - * WORKAROUNDS (Sandboxes API), delete each part once the platform covers it: - * - Commands run in `/work` by default, and a sandbox won't start a process in - * a directory that doesn't exist, so it's created here. `checkout()` also - * makes it, but a job doesn't have to check out. - * - Sandboxes boot with the loopback interface down, so nothing can listen on - * or reach `127.0.0.1`, which breaks services started with `.background()` - * and `waitForHttp`/`waitForPort`. Bringing it up is a no-op once the - * platform does it itself. - */ -export const machineSetupScript = `mkdir -p ${defaultCwd} && (ip link set lo up 2>/dev/null || true)`; - -/** - * The stages a machine is started in. Only the first is tried every time: - * the rest follow a snapshot that won't start. Each has its own step ID, - * machine name and trace name, all derived from the key. - */ -const stages = { - first: traceName.createMachine, - restart: traceName.restartMachine, - retry: traceName.retryCreate, - fresh: traceName.createFreshMachine, -}; - -type Stage = keyof typeof stages; - -type Sandbox = MachineHandle["sandbox"]; - -/** - * Create this scope's machine, from the snapshot it starts from if it has one. - * - * A snapshot that won't start is retried once, and a second failure makes it - * broken. Either way the job doesn't fail: the parent is rebuilt for it to - * start from, and a fresh machine is the last resort. - */ -export const startMachine = async ( - scope: CiJobScope, - /** The first stage's step ID, which the other stages build on. */ - stepId: string, -): Promise => { - const sandbox = await startSandbox(scope, stepId); - - return { - sandbox, - name: machineName(scope.run.runId, scope.path), - id: sandbox.id, - }; -}; - -const startSandbox = async ( - scope: CiJobScope, - stepId: string, -): Promise => { - const { run } = scope; - const tools = run.sandboxTools; - - if (!tools) { - throw new CiUsageError( - "This pipeline has no `step.sandbox` tools. `createCi` adds `sandboxMiddleware()` to the functions it creates, so this usually means the function was created another way.", - ); - } - - const machineConfig = resolveMachineConfig( - scope.config.machine ?? run.machine ?? run.ci.defaultMachine, - ); - - const idOf = (stage: Stage): string => { - return stage === "first" ? stepId : joinId(stepId, stage); - }; - - const nameOf = (stage: Stage): string => { - return machineName( - run.runId, - stage === "first" ? scope.path : `${scope.path} ${stage}`, - ); - }; - - /** Create a machine at a stage, from a snapshot if one is given, and set it up. */ - const start = async (stage: Stage, snapshotId?: string): Promise => { - const step = { id: idOf(stage), name: stages[stage] }; - const name = nameOf(stage); - - const sandbox = await tools.create( - step, - snapshotId ? { name, snapshotId } : { name, ...machineConfig }, - ); - - await sandbox.commands.run( - { id: joinId(step.id, "setup"), name: traceName.prepareWorkspace }, - ["/bin/sh", "-c", machineSetupScript], - ); - - return sandbox; - }; - - /** - * Start from a snapshot, or the start failure that says it won't. The failed - * start still holds its name, so what follows gets another, and the stuck - * machine is cleared away meanwhile. - */ - const restore = async ( - stage: Stage, - snapshotId: string, - ): Promise<{ started: Sandbox } | { failure: unknown }> => { - try { - return { started: await start(stage, snapshotId) }; - } catch (failure) { - if (!isStartFailure(failure)) { - throw failure; - } - - await discardFailedStart(run, idOf(stage), nameOf(stage), failure); - - return { failure }; - } - }; - - const startFresh = (note: string): Promise => { - scope.fromSnapshotId = undefined; - scope.startNote = note; - scope.restoreFallback = scope.rebuildParent; - - run.ci.hooks.activity(run, scope.jobPath, note); - - return start("fresh"); - }; - - run.ci.hooks.activity( - run, - scope.jobPath, - scope.startNote ?? "creating machine…", - ); - - const snapshotId = scope.fromSnapshotId; - - if (!snapshotId) { - return start("first"); - } - - const parentId = scope.fromJobIds[0] ?? "the parent"; - - // Every job starting from a snapshot checks it for itself, so what a job - // does never depends on whether a sibling got there first. Only the - // rebuild of the parent, which each of them awaits, is shared. - const first = await restore("first", snapshotId); - - if ("started" in first) { - return first.started; - } - - /** Why the snapshot can't be used. */ - let bad = `wouldn't start (${errorMessage(first.failure)})`; - - /** How to replace it. */ - let rebuild = { broken: false, unnamed: false }; - - // WORKAROUND (Sandboxes API): a snapshot can't be told to be broken, so - // it is retried before it is deleted. Delete this once snapshots or - // images report their own health. Today a snapshot has only the - // statuses `CREATING`, `READY`, `DELETING` and `DELETED`, and a broken - // one still says `READY`. A start that times out says no more: the - // machine may have been stuck, or the node busy, as easily as the - // snapshot bad. Deleting on the first one would let a single blip - // destroy a cache that is shared, since a pull request restores its - // base branch's snapshots, and every run after it would build again. So - // only a snapshot that fails to start a second time is broken enough to - // delete. Capacity errors (`compute_unavailable`, 429 and 503) are not - // start failures at all, and fail the job as any other would. - const state = await snapshotState( - run, - joinId(stepId, "snapshot-state"), - snapshotId, - ); - - if (state === "creating") { - // Another build is still taking it and may yet finish, so this - // run rebuilds without contending for its name. - rebuild = { broken: false, unnamed: true }; - } else if (state === "ready") { - // With the default wait, the same as the first try's: a shorter one - // would make a slow node more likely to fail the retry, and a second - // failure deletes the snapshot. - const second = await restore("restart", snapshotId); - - if ("started" in second) { - run.warnings.push( - `retried: snapshot of \`${parentId}\` needed a retry to start (${errorMessage(first.failure)}), so \`${scope.path}\` started it on the second try`, - ); - - return second.started; - } - - bad = `wouldn't start twice (${errorMessage(second.failure)})`; - - rebuild = { broken: true, unnamed: false }; - } - - run.warnings.push( - `fell back: snapshot of \`${parentId}\` ${bad}, so \`${scope.path}\` rebuilt it`, - ); - - const note = `rebuilding ${parentId} · bad snapshot`; - - scope.startNote = note; - - run.ci.hooks.activity(run, scope.jobPath, note); - - // Shared: jobs that had the same trouble wait for one rebuild, which - // also deletes a broken snapshot once. - const replacement = await scope.rebuildSnapshot?.(rebuild); - - if (replacement) { - scope.startNote = `starting ${parentId}`; - - run.ci.hooks.activity(run, scope.jobPath, scope.startNote); - - // Restores may still be broken while fresh machines work, which is where - // this job went before there was a replacement to try. - const retried = await restore("retry", replacement); - - if ("started" in retried) { - return retried.started; - } - } - - return startFresh(note); -}; - -/** - * Best-effort cleanup of a machine that failed to start. The platform keeps - * such a machine (stuck in STARTING) and its name, so it is destroyed here, and - * the end of the run finds it by its name if this couldn't. - * - * Never throws: a machine that can't be cleaned up now must not fail the job. - */ -const discardFailedStart = async ( - run: CiRunScope, - stepId: string, - name: string, - error: unknown, -): Promise => { - const knownId = (error as { cause?: { sandboxId?: string } })?.cause - ?.sandboxId; - - try { - await run.step.run( - { - id: joinId(stepId, "discard"), - name: traceName.discardMachine, - }, - async (): Promise<{ id?: string }> => { - // Errors are swallowed inside the step so it never retries. - let id = knownId; - - try { - let cursor: string | undefined; - - while (!id) { - const page = await run.ci.client.sandboxes.list({ - ...(cursor ? { cursor } : {}), - limit: 100, - }); - - id = page.items.find( - (sandbox: { name: string; status: string }) => { - return sandbox.name === name && sandbox.status !== "TERMINATED"; - }, - )?.id; - - if (id || !page.page.hasMore) { - break; - } - - cursor = page.page.cursor; - } - - if (id) { - const sandbox = await run.ci.client.sandboxes.get(id); - - await sandbox?.destroy(); - } - } catch { - // The run's cleanup finds it again by its name. - } - - return id ? { id } : {}; - }, - ); - } catch { - // Best effort only. - } -}; diff --git a/packages/ci/src/pipeline/cacheGuarantees.test.ts b/packages/ci/src/pipeline/cacheGuarantees.test.ts index 8814edda2..c34aa0b88 100644 --- a/packages/ci/src/pipeline/cacheGuarantees.test.ts +++ b/packages/ci/src/pipeline/cacheGuarantees.test.ts @@ -18,7 +18,7 @@ import { describe, expect, test } from "vitest"; import { consoleReporter } from "../github/auth.ts"; import { $ } from "../machine/command.ts"; -import { machineSetupScript } from "../machine/start.ts"; +import { machineSetupScript } from "../machine/machine.ts"; import { createCiTestClient } from "../testing/client.ts"; import { prEvent, prTrigger } from "../testing/events.ts"; import { createFakeSandboxApi } from "../testing/fakeSandbox.ts"; diff --git a/packages/ci/src/pipeline/pipeline.test.ts b/packages/ci/src/pipeline/pipeline.test.ts index caa4f65fa..27d0b17b6 100644 --- a/packages/ci/src/pipeline/pipeline.test.ts +++ b/packages/ci/src/pipeline/pipeline.test.ts @@ -22,9 +22,12 @@ import { import { consoleReporter, githubToken } from "../github/auth.ts"; import { github } from "../github/index.ts"; import { $ } from "../machine/command.ts"; -import { destroyOrphans, destroyRunMachines } from "../machine/machine.ts"; +import { + destroyOrphans, + destroyRunMachines, + machineSetupScript, +} from "../machine/machine.ts"; import { sandbox } from "../machine/sandbox.ts"; -import { machineSetupScript } from "../machine/start.ts"; import { report } from "../report.ts"; import { createCiTestClient } from "../testing/client.ts"; import { prEvent, prTrigger } from "../testing/events.ts"; From 7f9092bab9508f7bfd363bd616a7afe611b307c2 Mon Sep 17 00:00:00 2001 From: x Date: Sat, 10 Oct 2026 00:45:28 +0000 Subject: [PATCH 04/18] refactor(ci): require inngest 4.23.1 and drop the older-SDK paths The span API and a snapshot's own span are always there, so the stub, the version gate and the dual snapshot-span path are gone. The shared test harness had one user, so it is local to that test again. Co-Authored-By: Claude Opus 5.5 --- .changeset/ci-trace-spans.md | 4 +- packages/ci/package.json | 2 +- packages/ci/src/machine/machine.ts | 14 +- packages/ci/src/machine/spans.test.ts | 108 +++++++++++- packages/ci/src/pipeline/README.md | 2 +- packages/ci/src/pipeline/spans.test.ts | 218 ++----------------------- packages/ci/src/pipeline/spans.ts | 92 +---------- packages/ci/src/testing/README.md | 2 - packages/ci/src/testing/harness.ts | 115 ------------- packages/ci/src/testing/runFunction.ts | 14 +- packages/ci/src/testing/setup.ts | 11 -- packages/ci/src/testing/spanStub.ts | 185 --------------------- packages/ci/vitest.config.ts | 1 - 13 files changed, 140 insertions(+), 628 deletions(-) delete mode 100644 packages/ci/src/testing/harness.ts delete mode 100644 packages/ci/src/testing/setup.ts delete mode 100644 packages/ci/src/testing/spanStub.ts diff --git a/.changeset/ci-trace-spans.md b/.changeset/ci-trace-spans.md index abe6dd96e..ac0d50c6a 100644 --- a/.changeset/ci-trace-spans.md +++ b/.changeset/ci-trace-spans.md @@ -2,4 +2,6 @@ "@inngest/ci": minor --- -Group a run's steps into trace spans (one per job, one per command, one per extra machine), name CI's own steps for what they do, and mark them as CI's work so a trace can tell them from yours. Give a job a `name` to change what its span is called. Every span has a kind (job, sandbox, command, snapshot, github or attempt), and each statement on a background process (`output`, `exited`, `kill`) is a row of its own. Saving a sandbox is one `snapshot` row on an `inngest` that takes the span, and the span is opened by CI on older ones. +Group a run's steps into trace spans (one per job, one per command, one per extra machine), name CI's own steps for what they do, and mark them as CI's work so a trace can tell them from yours. Give a job a `name` to change what its span is called. Every span has a kind (job, sandbox, command, snapshot, github or attempt), and each statement on a background process (`output`, `exited`, `kill`) is a row of its own. Saving a sandbox is one `snapshot` row. + +`@inngest/ci` now needs `inngest` 4.23.1 or later, which takes a snapshot's own trace span and always has the span API; older SDKs are no longer supported. diff --git a/packages/ci/package.json b/packages/ci/package.json index 045e77192..0028cd97e 100644 --- a/packages/ci/package.json +++ b/packages/ci/package.json @@ -64,7 +64,7 @@ "node": ">=20" }, "peerDependencies": { - "inngest": "^4.22.0" + "inngest": "^4.23.1" }, "dependencies": { "@octokit/auth-app": "^8.3.1", diff --git a/packages/ci/src/machine/machine.ts b/packages/ci/src/machine/machine.ts index 8d9751a38..9a36a46f1 100644 --- a/packages/ci/src/machine/machine.ts +++ b/packages/ci/src/machine/machine.ts @@ -23,11 +23,7 @@ import { runInScope, scopeSeparator, } from "../pipeline/scope.ts"; -import { - inSnapshotSpan, - inSpan, - snapshotSpanOption, -} from "../pipeline/spans.ts"; +import { inSpan } from "../pipeline/spans.ts"; import type { MachineConfig } from "../types.ts"; import { boundedName, @@ -501,13 +497,13 @@ export interface TakenSnapshot { /** * A snapshot step: its ID is the step's, its name reads as an action. Saving * a job's sandbox is one statement, so its row is the job's "Save sandbox" - * span, which the SDK is told of where it can take it. + * span, which the SDK takes as the step's own so it opens no second group. */ const snapshotStep = (jobPath: string, id: string) => { return { id, name: traceName.snapshotMachine, - ...snapshotSpanOption(spans.save(jobPath)), + "~span": spans.save(jobPath), }; }; @@ -532,9 +528,7 @@ export const snapshotMachine = async ( const handle = await scope.machine; - return inSnapshotSpan(spans.save(scope.path), () => { - return takeSnapshot(run, scope.path, handle, cache); - }); + return takeSnapshot(run, scope.path, handle, cache); }; const takeSnapshot = async ( diff --git a/packages/ci/src/machine/spans.test.ts b/packages/ci/src/machine/spans.test.ts index b88bbaf4d..91847ca11 100644 --- a/packages/ci/src/machine/spans.test.ts +++ b/packages/ci/src/machine/spans.test.ts @@ -10,17 +10,119 @@ * @module */ -import { step } from "inngest"; +import { version as sdkVersion, step } from "inngest"; import { describe, expect, test } from "vitest"; import { CommandFailedError } from "../errors.ts"; +import { consoleReporter } from "../github/auth.ts"; +import { createCi } from "../pipeline/createCi.ts"; import { ciOrigin } from "../pipeline/names.ts"; +import { createCiTestClient } from "../testing/client.ts"; +import { prEvent, prTrigger } from "../testing/events.ts"; +import type { CommandScript, FakeSandboxApi } from "../testing/fakeSandbox.ts"; import { createFakeSandboxApi } from "../testing/fakeSandbox.ts"; -import type { Ci } from "../testing/harness.ts"; -import { ciTest, drawTrace } from "../testing/harness.ts"; +import { runFunction } from "../testing/runFunction.ts"; import { version } from "../version.ts"; import { $ } from "./command.ts"; import { sandbox } from "./sandbox.ts"; +type Ci = ReturnType; + +type RunResult = Awaited>; + +const ciTest = ( + options: { + /** The fake API to run on. Defaults to a fresh one. */ + api?: FakeSandboxApi; + /** What the commands it runs do. */ + scripts?: CommandScript[]; + } = {}, +) => { + const api = options.api ?? createFakeSandboxApi(); + + api.script(options.scripts ?? []); + + const ci = createCi(createCiTestClient(api), { github: consoleReporter() }); + + return { + api, + ci, + /** Run the pipeline whose handler `define` makes from `ci`. */ + run: (define: (ci: Ci) => () => Promise): Promise => { + const pipeline = ci.pipeline({ id: "pr", on: prTrigger }, define(ci)); + + return runFunction(pipeline, { event: prEvent }); + }, + }; +}; + +interface TraceNode { + label: string; + children: Map; +} + +/** + * Draw a run's trace as an indented tree, the way the UI nests it: each span + * once, where its first step is, with its kind in brackets, and each step + * under its spans. + */ +const drawTrace = ( + result: RunResult, + /** Suffix each row with who it says did it, as in `Create sandbox <- ci`. */ + withOrigins = false, +): string => { + const root: TraceNode = { label: "", children: new Map() }; + + const by = (origin: string | undefined) => { + if (!withOrigins || !origin) { + return ""; + } + + const names: Record = { + [ciOrigin]: "ci", + [`inngest@${sdkVersion}`]: "inngest", + }; + + return ` <- ${names[origin] ?? origin}`; + }; + + for (const stepId of result.stepIds) { + let node = root; + + for (const span of result.spans[stepId] ?? []) { + const key = `span:${span.id}`; + const label = span.kind ? `${span.name} [${span.kind}]` : span.name; + + const child = node.children.get(key) ?? { + label: `${label}${by(span.origin)}`, + children: new Map(), + }; + + node.children.set(key, child); + + node = child; + } + + node.children.set(`step:${stepId}`, { + label: `${result.names[stepId] ?? stepId}${by(result.origins[stepId])}`, + children: new Map(), + }); + } + + const lines: string[] = []; + + const draw = (node: TraceNode, depth: number) => { + for (const child of node.children.values()) { + lines.push(`${" ".repeat(depth)}${child.label}`); + + draw(child, depth + 1); + } + }; + + draw(root, 0); + + return lines.join("\n"); +}; + /** Run a job with a captured, a managed, a retried and a background command. */ const run = () => { return ciTest({ diff --git a/packages/ci/src/pipeline/README.md b/packages/ci/src/pipeline/README.md index d6748126e..7072e7007 100644 --- a/packages/ci/src/pipeline/README.md +++ b/packages/ci/src/pipeline/README.md @@ -11,6 +11,6 @@ The client and the run engine. - `metadata.ts`: the `userland.inngest-ci` metadata attached to runs and steps, and the helpers that build and attach it. - `scope.ts`: the run and job scopes held in async context. Jobs only write to the run scope, except for `builds`, the promises of the builds `from` parents need. - `names.ts`: what each step and span CI writes is called in the trace, and the origin that marks CI's own work. -- `spans.ts`: the one place CI touches the SDK's experimental trace-span API. It is feature-detected, so CI runs unchanged on an SDK without it. +- `spans.ts`: the one place CI touches the SDK's experimental trace-span API. - `durable.ts`: durable proxies that run calls as steps. - `rerun.ts`: re-running a pipeline from a GitHub check. diff --git a/packages/ci/src/pipeline/spans.test.ts b/packages/ci/src/pipeline/spans.test.ts index 100f1de8c..6e5f13b71 100644 --- a/packages/ci/src/pipeline/spans.test.ts +++ b/packages/ci/src/pipeline/spans.test.ts @@ -1,131 +1,16 @@ /** - * Tests of the span helpers, and of CI on an SDK without the span API: with - * none, functions still run, step IDs and names are exactly what they are - * with it, and the trace just has no spans or origins. The same goes for an - * SDK that doesn't take a snapshot's own span. The rest of the suite runs - * with the test stub in place. + * Tests of the span helpers, and of the one span a snapshot is: CI hands it + * to the SDK as the step's own, so the SDK opens no second group around the + * snapshot's create and wait. * * @module */ -import { group, step } from "inngest"; -import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -import { $ } from "../machine/command.ts"; +import { describe, expect, test, vi } from "vitest"; import { snapshotMachine } from "../machine/machine.ts"; -import { ciTest } from "../testing/harness.ts"; -import { installSpanStub, removeSpanStub } from "../testing/spanStub.ts"; import { ciOrigin } from "./names.ts"; import type { CiJobScope } from "./scope.ts"; -import { - hasSpanApi, - inSnapshotSpan, - inSpan, - originOption, - sdk, - snapshotSpanOption, - versionAtLeast, -} from "./spans.ts"; - -/** Run a job with a command and a step of your own. */ -const run = () => { - return ciTest().run((ci) => { - return ci.job("test", async () => { - await $`pnpm test`; - - await step.run("mine", () => { - return "done"; - }); - }); - }); -}; - -const span = { id: "a", name: "A", kind: "job" } as const; - -describe("without the SDK's span API", () => { - beforeEach(() => { - removeSpanStub(); - }); - - afterEach(() => { - installSpanStub(); - }); - - test("is detected as absent", () => { - expect(hasSpanApi()).toBe(false); - - expect("~span" in group).toBe(false); - }); - - test("inSpan runs the function and returns what it returns", async () => { - expect(inSpan(span, () => 7)).toBe(7); - - await expect( - inSpan(span, () => { - return Promise.resolve("later"); - }), - ).resolves.toBe("later"); - - expect(() => { - return inSpan(span, () => { - throw new Error("from fn"); - }); - }).toThrow("from fn"); - }); - - test("steps keep their IDs and names, and the trace has no spans", async () => { - installSpanStub(); - - const withSpans = await run(); - - removeSpanStub(); - - const without = await run(); - - expect(without.type).toBe("function-resolved"); - - expect(without.stepIds).toEqual(withSpans.stepIds); - expect(without.names).toEqual(withSpans.names); - - expect(without.stepIds).toContain("test › pnpm test › start"); - expect(without.names["test › pnpm test › start"]).toBe("Start process"); - - expect(Object.keys(withSpans.spans).length).toBeGreaterThan(0); - expect(without.spans).toEqual({}); - - // Origins aren't asserted absent: an SDK with the span API stamps a - // step's `~origin` option itself, whether or not `group["~span"]` is there - // to open spans, and this suite runs against that SDK. - }); -}); - -describe("with the SDK's span API", () => { - test("is detected, and inSpan hands it the span and the function", () => { - expect(hasSpanApi()).toBe(true); - - removeSpanStub(); - - const calls: unknown[] = []; - - Object.defineProperty(group, "~span", { - value: (opened: unknown, fn: () => unknown) => { - calls.push(opened); - - return `wrapped ${String(fn())}`; - }, - configurable: true, - }); - - try { - expect(inSpan(span, () => "inner")).toBe("wrapped inner"); - - expect(calls).toEqual([span]); - } finally { - Reflect.deleteProperty(group, "~span"); - - installSpanStub(); - } - }); -}); +import { originOption } from "./spans.ts"; describe("originOption", () => { test("is an extra key on the step options, and nothing else", () => { @@ -137,39 +22,9 @@ describe("originOption", () => { }); }); -describe("versionAtLeast", () => { - test.each([ - ["4.23.1", true], - ["4.23.2", true], - ["4.24.0", true], - ["5.0.0", true], - ["4.23.1-pr.1", true], - ["4.23.0", false], - ["4.22.9", false], - ["3.99.99", false], - ])("%s", (version, expected) => { - expect(versionAtLeast(version, "4.23.1")).toBe(expected); - }); -}); - describe("a snapshot's span", () => { - /** What a job's snapshot passes the SDK, and the spans opened around it. */ - const snapshotWith = async (takesSpan: boolean) => { - vi.spyOn(sdk, "takesSnapshotSpan").mockReturnValue(takesSpan); - + test("is the job's Save sandbox span, passed as the step's own", async () => { const steps: unknown[] = []; - const opened: unknown[] = []; - - const open = vi.fn((spanned: unknown, fn: () => unknown) => { - opened.push(spanned); - - return fn(); - }); - - Object.defineProperty(group, "~span", { - value: open, - configurable: true, - }); const scope = { path: "test", @@ -185,59 +40,18 @@ describe("a snapshot's span", () => { run: { ci: { hooks: { activity: vi.fn() } } }, } as unknown as CiJobScope; - try { - await snapshotMachine(scope); - } finally { - Reflect.deleteProperty(group, "~span"); - - installSpanStub(); - - vi.restoreAllMocks(); - } - - return { steps, opened }; - }; - - const save = { - id: "test › save", - name: "Save sandbox", - kind: "snapshot", - origin: ciOrigin, - }; - - test("goes to an SDK that takes it, as the step's own span", async () => { - removeSpanStub(); - - const { steps, opened } = await snapshotWith(true); + await snapshotMachine(scope); expect(steps).toEqual([ - expect.objectContaining({ id: "test › snapshot", "~span": save }), + expect.objectContaining({ + id: "test › snapshot", + "~span": { + id: "test › save", + name: "Save sandbox", + kind: "snapshot", + origin: ciOrigin, + }, + }), ]); - - expect(opened).toEqual([]); - }); - - test("is opened here for an SDK that doesn't, which would open its own", async () => { - removeSpanStub(); - - const { steps, opened } = await snapshotWith(false); - - expect(steps).toEqual([expect.not.objectContaining({ "~span": save })]); - - expect(opened).toEqual([save]); - }); - - test("is a step option only where the SDK takes it", () => { - vi.spyOn(sdk, "takesSnapshotSpan").mockReturnValue(true); - - expect(snapshotSpanOption(span)).toEqual({ "~span": span }); - - vi.spyOn(sdk, "takesSnapshotSpan").mockReturnValue(false); - - expect(snapshotSpanOption(span)).toEqual({}); - - expect(inSnapshotSpan(span, () => "ran")).toBe("ran"); - - vi.restoreAllMocks(); }); }); diff --git a/packages/ci/src/pipeline/spans.ts b/packages/ci/src/pipeline/spans.ts index b2471c28e..57ad8e745 100644 --- a/packages/ci/src/pipeline/spans.ts +++ b/packages/ci/src/pipeline/spans.ts @@ -1,16 +1,13 @@ /** - * The one place CI touches the SDK's experimental trace-span API. The span - * API ships in newer SDKs than the oldest one CI supports, so it's - * feature-detected here: where the SDK has it, spans and origins mark up the - * trace, and where it doesn't, `inSpan` just runs the function and the origin - * step option is an unknown key the SDK ignores. Step IDs and names never - * depend on it. Everything else in CI goes through these helpers rather than - * reaching for the SDK's span API itself. + * The one place CI touches the SDK's experimental trace-span API: spans group + * the steps run inside them in the trace, and an origin marks work as CI's. + * Step IDs and names never depend on either. Everything else in CI goes + * through these helpers rather than reaching for the SDK's span API itself. * * @module */ -import { group, version as sdkVersion } from "inngest"; +import { group } from "inngest"; /** What a span stands for. Every span CI opens has one. */ export type SpanKind = @@ -33,93 +30,20 @@ export interface SpanInfo { origin?: string; } -/** The part of the SDK's `group` this file uses, when the SDK has it. */ -interface SpanApi { - "~span"?: (span: SpanInfo, fn: () => R) => R; -} - -const spanApi = (): SpanApi => { - return group as unknown as SpanApi; -}; - -/** Whether the SDK in use has the span API. */ -export const hasSpanApi = (): boolean => { - return typeof spanApi()["~span"] === "function"; -}; - -/** - * Run `fn` in a trace span, or just run it when the SDK has no span API. - */ +/** Run `fn` in a trace span. */ export const inSpan = ( /** The span to run `fn` in. */ span: SpanInfo, /** What to run. */ fn: () => R, ): R => { - const api = spanApi(); - const open = api["~span"]; - - if (typeof open !== "function") { - return fn(); - } - - return open.call(api, span, fn) as R; + return group["~span"](span, fn); }; -/** - * The step option that marks a step as work `origin` did. A step option is - * just a key on the options object, and an SDK without the span API ignores - * it, so it's safe to always pass. - */ +/** The step option that marks a step as work `origin` did. */ export const originOption = ( /** Who did the work. */ origin: string, ): { "~origin": string } => { return { "~origin": origin }; }; - -/** The first SDK whose snapshot takes a caller's `"~span"` as its own span. */ -const snapshotSpanSince = "4.23.1"; - -/** Whether `version` is `since` or later, as in `4.23.1`. */ -export const versionAtLeast = (version: string, since: string): boolean => { - const have = version.split(/[.-]/).map(Number); - - for (const [index, want] of since.split(".").map(Number).entries()) { - const got = have[index] ?? 0; - - if (got !== want) { - return got > want; - } - } - - return true; -}; - -/** - * What the SDK in use can do beyond the span API. Older SDKs ignore the step - * options they don't know, so each is detected here rather than assumed. - */ -export const sdk = { - /** - * Whether a snapshot step's `"~span"` option stands in for the span the SDK - * would open around its create and wait. Older SDKs ignore the option and - * open their own, which then sits inside the span CI opens. - */ - takesSnapshotSpan: (): boolean => { - return versionAtLeast(sdkVersion, snapshotSpanSince); - }, -}; - -/** - * The step option that makes a snapshot one statement in `span`, where the - * SDK takes one. Elsewhere there is none, and `inSnapshotSpan` opens it. - */ -export const snapshotSpanOption = (span: SpanInfo): { "~span"?: SpanInfo } => { - return sdk.takesSnapshotSpan() ? { "~span": span } : {}; -}; - -/** Run `fn`, which takes snapshots, in `span` unless the SDK does it itself. */ -export const inSnapshotSpan = (span: SpanInfo, fn: () => R): R => { - return sdk.takesSnapshotSpan() ? fn() : inSpan(span, fn); -}; diff --git a/packages/ci/src/testing/README.md b/packages/ci/src/testing/README.md index bcbca1697..688f4b139 100644 --- a/packages/ci/src/testing/README.md +++ b/packages/ci/src/testing/README.md @@ -6,7 +6,5 @@ Test-only helpers. Not exported and not built. - `fakeGitHub.ts`: a fake GitHub HTTP layer. - `client.ts`: an Inngest client wired to the fake sandbox API, which also records the functions it creates. - `events.ts`: the pull request event and trigger tests share. -- `spanStub.ts`, `setup.ts`: a stand-in for the SDK's span API, installed before every test file when the SDK has none, so span and origin assertions hold on any SDK. -- `harness.ts`: `ciTest`, a client on the fake API with a `run` for a pipeline a test defines, and `drawTrace` for the trace it left. - `runFunction.ts`: drives a function to completion like the executor, including `step.invoke` of the functions the test client created. Invokes run beside the function, and the function is called again as each one ends; runs of one `event.data` concurrency key take turns. - `schema.ts`: a fake Standard Schema that can also write itself as JSON Schema. diff --git a/packages/ci/src/testing/harness.ts b/packages/ci/src/testing/harness.ts deleted file mode 100644 index 4fa5142c2..000000000 --- a/packages/ci/src/testing/harness.ts +++ /dev/null @@ -1,115 +0,0 @@ -/** - * What a test of a pipeline starts from: a client on the fake Sandboxes API - * and a `ci` that reports to the console, with `run` driving one pipeline - * whose handler a test defines. `drawTrace` draws the trace such a run left. - * - * @module - */ - -import { version as sdkVersion } from "inngest"; -import { consoleReporter } from "../github/auth.ts"; -import { createCi } from "../pipeline/createCi.ts"; -import { ciOrigin } from "../pipeline/names.ts"; -import { createCiTestClient } from "./client.ts"; -import { prEvent, prTrigger } from "./events.ts"; -import type { CommandScript, FakeSandboxApi } from "./fakeSandbox.ts"; -import { createFakeSandboxApi } from "./fakeSandbox.ts"; -import { runFunction } from "./runFunction.ts"; - -export type Ci = ReturnType; - -export type RunResult = Awaited>; - -export const ciTest = ( - options: { - /** The fake API to run on. Defaults to a fresh one. */ - api?: FakeSandboxApi; - /** What the commands it runs do. */ - scripts?: CommandScript[]; - } = {}, -) => { - const api = options.api ?? createFakeSandboxApi(); - - api.script(options.scripts ?? []); - - const ci = createCi(createCiTestClient(api), { github: consoleReporter() }); - - return { - api, - ci, - /** Run the pipeline whose handler `define` makes from `ci`. */ - run: (define: (ci: Ci) => () => Promise): Promise => { - const pipeline = ci.pipeline({ id: "pr", on: prTrigger }, define(ci)); - - return runFunction(pipeline, { event: prEvent }); - }, - }; -}; - -interface TraceNode { - label: string; - children: Map; -} - -/** - * Draw a run's trace as an indented tree, the way the UI nests it: each span - * once, where its first step is, with its kind in brackets, and each step - * under its spans. - */ -export const drawTrace = ( - result: RunResult, - /** Suffix each row with who it says did it, as in `Create sandbox <- ci`. */ - withOrigins = false, -): string => { - const root: TraceNode = { label: "", children: new Map() }; - - const by = (origin: string | undefined) => { - if (!withOrigins || !origin) { - return ""; - } - - const names: Record = { - [ciOrigin]: "ci", - [`inngest@${sdkVersion}`]: "inngest", - }; - - return ` <- ${names[origin] ?? origin}`; - }; - - for (const stepId of result.stepIds) { - let node = root; - - for (const span of result.spans[stepId] ?? []) { - const key = `span:${span.id}`; - const label = span.kind ? `${span.name} [${span.kind}]` : span.name; - - const child = node.children.get(key) ?? { - label: `${label}${by(span.origin)}`, - children: new Map(), - }; - - node.children.set(key, child); - - node = child; - } - - node.children.set(`step:${stepId}`, { - label: `${result.names[stepId] ?? stepId}${by(result.origins[stepId])}`, - children: new Map(), - }); - } - - const lines: string[] = []; - - const draw = (node: TraceNode, depth: number) => { - for (const child of node.children.values()) { - lines.push(`${" ".repeat(depth)}${child.label}`); - - draw(child, depth + 1); - } - }; - - draw(root, 0); - - return lines.join("\n"); -}; diff --git a/packages/ci/src/testing/runFunction.ts b/packages/ci/src/testing/runFunction.ts index 52b6f1085..29c461508 100644 --- a/packages/ci/src/testing/runFunction.ts +++ b/packages/ci/src/testing/runFunction.ts @@ -15,7 +15,6 @@ import type { EventPayload, InngestFunction } from "inngest"; import { createdFunctions } from "./client.ts"; -import { SpanStubMiddleware, spanStubInstalled, stampOf } from "./spanStub.ts"; /** The opcodes the harness reads; their values are the wire format. */ const StepOpCode = { @@ -340,12 +339,8 @@ export const runFunction = async ( steps[id] = step.data; - // Where the SDK has the span API it stamps these on the step. Where the - // test stub provides it, the stub recorded them instead. - const stamp = spanStubInstalled() ? stampOf(id) : undefined; - const span = - step.opts?.span ?? (stamp?.span.length ? stamp.span : undefined); - const origin = step.opts?.origin ?? stamp?.origin; + // The SDK stamps a step's spans and origin on it. + const { span, origin } = step.opts ?? {}; if (span) { spans[id] = span; @@ -580,11 +575,6 @@ const runOnce = async ( return new Middleware({ client }); }); - if (spanStubInstalled()) { - // First, so the other middleware builds on step tools that record. - middlewareInstances.unshift(new SpanStubMiddleware({ client })); - } - const execution = internals["createExecution"]({ partialOptions: { client, diff --git a/packages/ci/src/testing/setup.ts b/packages/ci/src/testing/setup.ts deleted file mode 100644 index f7532a0aa..000000000 --- a/packages/ci/src/testing/setup.ts +++ /dev/null @@ -1,11 +0,0 @@ -/** - * Runs before every test file: gives the SDK the span API when it has none, - * so span and origin assertions hold against any SDK. A test of the no-span - * path takes it off again with `removeSpanStub`. - * - * @module - */ - -import { installSpanStub } from "./spanStub.ts"; - -installSpanStub(); diff --git a/packages/ci/src/testing/spanStub.ts b/packages/ci/src/testing/spanStub.ts deleted file mode 100644 index 481979fbe..000000000 --- a/packages/ci/src/testing/spanStub.ts +++ /dev/null @@ -1,185 +0,0 @@ -/** - * A stand-in for the SDK's experimental trace-span API, for tests that run - * against an SDK that doesn't have it yet. It does what the real one does - * where tests can see it: a span opened with `inSpan` is kept in the async - * context, so a step called inside it is recorded under the span path, and a - * step's origin is its own `"~origin"` option or the innermost span's. Where - * the real SDK stamps a step's `opts.span` and `opts.origin` onto the step - * the executor reports, the stub records them by step ID for `runFunction` to - * read back. It installs nothing when the SDK has the span API itself. - * - * @module - */ - -import { group, Middleware } from "inngest"; -import type { AsyncContext } from "inngest/experimental"; -import { runWithAsyncCtx } from "inngest/experimental"; -import type { SpanInfo } from "../pipeline/spans.ts"; -import { hasSpanApi } from "../pipeline/spans.ts"; - -/** What the stub knows about a step: its spans, outermost first, and origin. */ -export interface StepStamp { - span: SpanInfo[]; - origin?: string; -} - -/** Where a span stack lives in an async context. Contexts copied by spread keep it. */ -const spanStackKey = Symbol("ci.spanStub.spans"); - -interface SpannedContext extends AsyncContext { - [spanStackKey]?: SpanInfo[]; -} - -const stamps = new Map(); - -let installed = false; - -/** - * The current async context, read synchronously like the real span API does. - * The SDK keeps its async storage on a global, and only offers an async getter, - * which would delay every span by a tick and reorder concurrent work. - */ -const currentContext = (): SpannedContext | undefined => { - const cache = ( - globalThis as Record< - symbol, - { resolved?: { getStore: () => AsyncContext | undefined } } | undefined - > - )[Symbol.for("inngest:als")]; - - return cache?.resolved?.getStore(); -}; - -const spanOf = (span: SpanInfo, fn: () => R): R => { - const ctx = currentContext(); - - if (!ctx) { - return fn(); - } - - const spans = [...(ctx[spanStackKey] ?? []), span]; - - const spanned: SpannedContext = { ...ctx, [spanStackKey]: spans }; - - return runWithAsyncCtx(spanned, fn); -}; - -/** The SDK's own `"~span"`, while `removeSpanStub` has it taken off. */ -let hidden: PropertyDescriptor | undefined; - -/** Give the SDK's `group` a `"~span"`, unless it already has one. */ -export const installSpanStub = (): void => { - if (hidden) { - Object.defineProperty(group, "~span", hidden); - - hidden = undefined; - - return; - } - - if (hasSpanApi()) { - return; - } - - Object.defineProperty(group, "~span", { - value: spanOf, - configurable: true, - writable: true, - }); - - installed = true; -}; - -/** - * Take the stub off again, so the SDK looks as it does without the span API. - * An SDK that has the span API itself has it taken off too, until - * `installSpanStub` puts it back. - */ -export const removeSpanStub = (): void => { - if (installed) { - Reflect.deleteProperty(group, "~span"); - - installed = false; - - return; - } - - hidden ??= Object.getOwnPropertyDescriptor(group, "~span"); - - Reflect.deleteProperty(group, "~span"); -}; - -/** Whether the stub is what's providing the span API. */ -export const spanStubInstalled = (): boolean => { - return installed; -}; - -/** What the stub recorded for a step, by the ID CI gave it. */ -export const stampOf = (stepId: string): StepStamp | undefined => { - return stamps.get(stepId); -}; - -/** Record the step `args` is about to run, from the span stack in effect now. */ -const record = (args: unknown[]): void => { - const first = args[0]; - - const options = - typeof first === "string" - ? { id: first } - : (first as { id?: unknown; "~origin"?: unknown } | undefined); - - if (typeof options?.id !== "string") { - return; - } - - const span = currentContext()?.[spanStackKey] ?? []; - - const inherited = [...span].reverse().find((entry) => { - return entry.origin; - })?.origin; - - const own = (options as { "~origin"?: unknown })["~origin"]; - - stamps.set(options.id, { - span, - origin: typeof own === "string" ? own : inherited, - }); -}; - -/** - * Wrap step tools so every call is recorded. The SDK's middleware hooks don't - * hand a step's unknown options to middleware, so the calls are watched where - * they're made. - */ -const recording = (tools: T): T => { - return new Proxy(tools, { - get(target, prop, receiver) { - const value = Reflect.get(target, prop, receiver) as unknown; - - if (typeof value === "function") { - return (...args: unknown[]) => { - record(args); - - return (value as (...a: unknown[]) => unknown).apply(target, args); - }; - } - - if (value && typeof value === "object") { - return recording(value); - } - - return value; - }, - }); -}; - -/** Records each step's span path and origin as it's called. */ -export class SpanStubMiddleware extends Middleware.BaseMiddleware { - readonly id = "ci-span-stub"; - - override transformFunctionInput( - arg: Middleware.TransformFunctionInputArgs, - ): Middleware.TransformFunctionInputArgs { - return { ...arg, ctx: { ...arg.ctx, step: recording(arg.ctx.step) } }; - } -} diff --git a/packages/ci/vitest.config.ts b/packages/ci/vitest.config.ts index d3b49e53c..2a6163f50 100644 --- a/packages/ci/vitest.config.ts +++ b/packages/ci/vitest.config.ts @@ -4,7 +4,6 @@ export default defineConfig({ test: { environment: "node", silent: "passed-only", - setupFiles: ["src/testing/setup.ts"], typecheck: { enabled: true, include: ["src/**/*.test.ts"], From 98f6ad1d50cc6d8b30fe98d57638909708e8bce6 Mon Sep 17 00:00:00 2001 From: x Date: Tue, 13 Oct 2026 17:10:43 +0000 Subject: [PATCH 05/18] refactor(ci): share step names, sandbox admin, snapshot names and cache lookups CI's step IDs and names move to one catalog in pipeline/names.ts, run through a thin ciRun helper that tags the step as before. Sandbox paging and snapshot deletes (machine/admin.ts), snapshot names (cache/names.ts), the cache lookup and the check-complete payload each have one implementation. No step ID, name or wire format changes. Co-Authored-By: Claude Opus 5.5 --- .changeset/ci-shared-helpers.md | 5 + packages/ci/src/cache/cache.ts | 132 +++++---- packages/ci/src/cache/names.ts | 25 ++ packages/ci/src/checkout/changed.ts | 12 +- packages/ci/src/checkout/checkout.ts | 11 +- packages/ci/src/github/checks.ts | 218 ++++++--------- packages/ci/src/github/helpers.ts | 7 +- packages/ci/src/machine/admin.test.ts | 70 +++++ packages/ci/src/machine/admin.ts | 75 +++++ packages/ci/src/machine/command.ts | 12 +- packages/ci/src/machine/machine.ts | 130 +++------ packages/ci/src/pipeline/durable.ts | 6 +- packages/ci/src/pipeline/job.ts | 31 +-- packages/ci/src/pipeline/metadata.ts | 34 ++- packages/ci/src/pipeline/names.ts | 301 +++++++++++++++++---- packages/ci/src/pipeline/pipeline.ts | 51 ++-- packages/ci/src/pipeline/stepNames.test.ts | 66 +++++ packages/ci/src/report.ts | 7 +- packages/ci/src/units.test.ts | 33 ++- 19 files changed, 792 insertions(+), 434 deletions(-) create mode 100644 .changeset/ci-shared-helpers.md create mode 100644 packages/ci/src/cache/names.ts create mode 100644 packages/ci/src/machine/admin.test.ts create mode 100644 packages/ci/src/machine/admin.ts create mode 100644 packages/ci/src/pipeline/stepNames.test.ts diff --git a/.changeset/ci-shared-helpers.md b/.changeset/ci-shared-helpers.md new file mode 100644 index 000000000..400014067 --- /dev/null +++ b/.changeset/ci-shared-helpers.md @@ -0,0 +1,5 @@ +--- +"@inngest/ci": patch +--- + +Internal refactor: CI's step IDs and names come from one catalog, and snapshot naming, sandbox paging and snapshot deletes, cache lookups and check updates share helpers. Nothing user-visible changes. diff --git a/packages/ci/src/cache/cache.ts b/packages/ci/src/cache/cache.ts index 173b327c5..59f072bdd 100644 --- a/packages/ci/src/cache/cache.ts +++ b/packages/ci/src/cache/cache.ts @@ -10,8 +10,9 @@ * @module */ -import { tagStep } from "../pipeline/metadata.ts"; -import { ciStep, traceName } from "../pipeline/names.ts"; +import { ciRun } from "../pipeline/metadata.ts"; +import type { StepSpec } from "../pipeline/names.ts"; +import { steps } from "../pipeline/names.ts"; import type { CiJobScope, CiRunScope } from "../pipeline/scope.ts"; import { countApi, joinId, rootRunIdOf } from "../pipeline/scope.ts"; import type { @@ -22,12 +23,12 @@ import type { RepoContext, } from "../types.ts"; import { - boundedName, formatRelative, hash, isSnapshotNotFound, stableStringify, } from "../util.ts"; +import { formatName } from "./names.ts"; /** * Where a job reads and writes its cached snapshots. @@ -82,19 +83,6 @@ const sharedScopes = ( return { read: [branch], write: branch }; }; -/** - * The name a job's cached snapshot has in a scope: `ci///`. - * A name too long for a snapshot keeps its start and gains a hash of the - * whole, so it stays unique. - */ -export const snapshotName = ( - scope: string, - jobId: string, - ownKey: string, -): string => { - return boundedName(`ci/${scope}/${jobId}/${ownKey}`); -}; - /** * Resolve the parts of a cache key into a single hash. * @@ -282,18 +270,18 @@ export const cacheTarget = async ( countApi("cache"); - const ownKey = (await run.step.run( - ciStep(joinId(job.path, "cache:key"), traceName.checkCache), - async () => { - await tagStep(run, { kind: "cache", job: job.path }); - - return jobCacheKey(run, cache, input, base); - }, - )) as string; + const ownKey = await ciRun(run, steps.cacheKey(job.path), () => { + return jobCacheKey(run, cache, input, base); + }); return { ownKey, - name: snapshotName(cacheScopes(run.repo, cache.scope).write, jobId, ownKey), + name: formatName({ + kind: "cache", + scope: cacheScopes(run.repo, cache.scope).write, + jobId, + ownKey, + }), }; }; @@ -314,7 +302,12 @@ export const runTarget = ( return { ownKey, - name: snapshotName(`run:${rootRunIdOf(run)}`, jobId, ownKey), + name: formatName({ + kind: "run", + rootRunId: rootRunIdOf(run), + jobId, + ownKey, + }), }; }; @@ -435,7 +428,7 @@ const findInScopes = async ( for (const scope of cacheScopes(run.repo, cache.scope).read) { const found = await findNamed( run, - snapshotName(scope, jobId, ownKey), + formatName({ kind: "cache", scope, jobId, ownKey }), exclude, ); @@ -459,24 +452,28 @@ export const lookupCache = async ( /** A snapshot found to be bad, which a rebuild must not find again. */ exclude?: string, ): Promise => { - const { run } = scope; - - const found = (await run.step.run( - ciStep(joinId(scope.path, "cache:lookup"), traceName.lookUpCache), - async () => { - await tagStep(run, { kind: "cache", job: scope.path }); - - const hit = await findCached( - run, - scope.config.id, - cache, - target, - exclude, - ); - - return hit ?? null; - }, - )) as CachedSnapshot | null; + return lookUp( + scope.run, + steps.lookUpCache(joinId(scope.path, "cache:lookup"), scope.path), + scope.config.id, + cache, + target, + exclude, + ); +}; + +/** A lookup as a memoized step: a hit is returned, a miss is `null` in the step. */ +const lookUp = async ( + run: CiRunScope, + spec: StepSpec, + jobId: string, + cache: CacheConfig | undefined, + target: CacheTarget, + exclude?: string, +): Promise => { + const found = await ciRun(run, spec, async () => { + return (await findCached(run, jobId, cache, target, exclude)) ?? null; + }); return found ?? undefined; }; @@ -515,18 +512,14 @@ export const lookupBeforeBuild = async ( /** A snapshot found to be bad, which a rebuild must not find again. */ exclude?: string, ): Promise => { - const found = (await run.step.run( - ciStep(joinId(job.stepPath, "lookup"), traceName.lookUpCache), - async () => { - await tagStep(run, { kind: "cache", job: job.path }); - - const hit = await findCached(run, job.id, cache, target, exclude); - - return hit ?? null; - }, - )) as CachedSnapshot | null; - - return found ?? undefined; + return lookUp( + run, + steps.lookUpCache(joinId(job.stepPath, "lookup"), job.path), + job.id, + cache, + target, + exclude, + ); }; /** @@ -546,9 +539,10 @@ export const resolveTakenName = async ( */ broken?: boolean, ): Promise<{ winner?: CachedSnapshot; cleared: boolean }> => { - return (await run.step.run( - ciStep(stepId, traceName.resolveCacheName), - async () => { + return ciRun( + run, + steps.resolveCacheName(stepId), + async (): Promise<{ winner?: CachedSnapshot; cleared: boolean }> => { const winner = await findNamed(run, name, exclude); if (winner) { @@ -582,7 +576,7 @@ export const resolveTakenName = async ( return { cleared }; }, - )) as { winner?: CachedSnapshot; cleared: boolean }; + ); }; /** @@ -596,9 +590,10 @@ export const snapshotState = async ( stepId: string, snapshotId: string, ): Promise<"gone" | "creating" | "ready"> => { - return (await run.step.run( - ciStep(stepId, traceName.checkSnapshotState), - async () => { + return ciRun( + run, + steps.checkSnapshotState(stepId), + async (): Promise<"gone" | "creating" | "ready"> => { try { const snapshot = (await snapshotsClient(run).get(snapshotId)) as | SnapshotResource @@ -614,7 +609,7 @@ export const snapshotState = async ( return "ready"; } }, - )) as "gone" | "creating" | "ready"; + ); }; /** @@ -628,8 +623,9 @@ export const deleteSnapshot = async ( snapshotId: string, ): Promise => { try { - const result = (await run.step.run( - ciStep(stepId, traceName.deleteBadSnapshot), + const result = await ciRun( + run, + steps.deleteBadSnapshot(stepId), async () => { try { const snapshot = await snapshotsClient(run).get(snapshotId); @@ -642,7 +638,7 @@ export const deleteSnapshot = async ( return { deleted: false, gone: isSnapshotNotFound(error) }; } }, - )) as { deleted: boolean; gone: boolean }; + ); // Outside the step, which a replay doesn't run: the set is rebuilt on // every replay, so a snapshot gone now must leave it on each of them. diff --git a/packages/ci/src/cache/names.ts b/packages/ci/src/cache/names.ts new file mode 100644 index 000000000..b3552f1b6 --- /dev/null +++ b/packages/ci/src/cache/names.ts @@ -0,0 +1,25 @@ +/** + * The names snapshots are found by. Everything that builds a snapshot's name + * goes through here, so a change to the naming (such as named images) is a + * change to this file. + * + * @module + */ + +import { boundedName } from "../util.ts"; + +/** What a snapshot's name says it is. */ +export type SnapshotName = + | { kind: "cache"; scope: string; jobId: string; ownKey: string } + | { kind: "run"; rootRunId: string; jobId: string; ownKey: string }; + +/** + * The name a snapshot has: `ci///`, where a run's own + * snapshots have the scope `run:`. A name too long for a snapshot + * keeps its start and gains a hash of the whole, so it stays unique. + */ +export const formatName = (name: SnapshotName): string => { + const scope = name.kind === "cache" ? name.scope : `run:${name.rootRunId}`; + + return boundedName(`ci/${scope}/${name.jobId}/${name.ownKey}`); +}; diff --git a/packages/ci/src/checkout/changed.ts b/packages/ci/src/checkout/changed.ts index 8fad40dce..b88829833 100644 --- a/packages/ci/src/checkout/changed.ts +++ b/packages/ci/src/checkout/changed.ts @@ -6,7 +6,8 @@ */ import { CiUsageError } from "../errors.ts"; -import { traceName } from "../pipeline/names.ts"; +import { ciRun } from "../pipeline/metadata.ts"; +import { steps } from "../pipeline/names.ts"; import type { CiRunScope } from "../pipeline/scope.ts"; import { countApi, getRunScope } from "../pipeline/scope.ts"; import type { RepoContext } from "../types.ts"; @@ -91,9 +92,10 @@ export const changedFiles = async (): Promise => { }; const readChangedFiles = async (run: CiRunScope): Promise => { - const files = (await run.step.run( - { id: "changed", name: traceName.findChangedFiles }, - async () => { + const files = await ciRun( + run, + steps.findChangedFiles(), + async (): Promise => { try { return await listChangedFiles(run.repo); } catch (error) { @@ -106,7 +108,7 @@ const readChangedFiles = async (run: CiRunScope): Promise => { return { unknown: true as const, reason: error.message }; } }, - )) as string[] | { unknown: true; reason: string }; + ); if (!Array.isArray(files)) { run.warnings.push( diff --git a/packages/ci/src/checkout/checkout.ts b/packages/ci/src/checkout/checkout.ts index cb8256309..e0c7198c1 100644 --- a/packages/ci/src/checkout/checkout.ts +++ b/packages/ci/src/checkout/checkout.ts @@ -7,7 +7,8 @@ import { CiUsageError } from "../errors.ts"; import { ensureMachine } from "../machine/machine.ts"; -import { traceName } from "../pipeline/names.ts"; +import { ciRun } from "../pipeline/metadata.ts"; +import { steps } from "../pipeline/names.ts"; import type { CiRunScope, MachineHandle } from "../pipeline/scope.ts"; import { countApi, @@ -91,11 +92,9 @@ export const checkout = async (opts: CheckoutOptions = {}): Promise => { const machine = await ensureMachine(scope); const stepId = joinId(scope.path, "checkout"); - await run.step.run( - { - id: stepId, - name: local ? traceName.uploadWorkingTree : traceName.cloneRepository, - }, + await ciRun( + run, + local ? steps.uploadWorkingTree(stepId) : steps.cloneRepository(stepId), async () => { if (local) { return uploadWorkingTree(run, machine, local.path, target); diff --git a/packages/ci/src/github/checks.ts b/packages/ci/src/github/checks.ts index f6a36c54e..e03d21515 100644 --- a/packages/ci/src/github/checks.ts +++ b/packages/ci/src/github/checks.ts @@ -5,8 +5,8 @@ * @module */ -import { type StepTag, tagStep } from "../pipeline/metadata.ts"; -import { traceName } from "../pipeline/names.ts"; +import { ciRun } from "../pipeline/metadata.ts"; +import { type StepSpec, steps } from "../pipeline/names.ts"; import type { CiRunScope } from "../pipeline/scope.ts"; import { inGitHubSpan } from "../pipeline/scope.ts"; import type { CheckAnnotation, CheckConclusion } from "../types.ts"; @@ -230,11 +230,12 @@ export const normaliseAnnotation = ( /** Run a check update as a step in the run's GitHub span. */ const githubStep = ( run: CiRunScope, - step: { id: string; name: string }, + spec: StepSpec, fn: () => Promise, + runValues?: () => Record, ): Promise => { return inGitHubSpan(run, () => { - return run.step.run(step, fn) as Promise; + return ciRun(run, spec, fn, runValues); }); }; @@ -271,21 +272,44 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return checkRunId === undefined ? {} : { checkRunId }; }; - const start = async ( + /** What `sink.complete` is given for a finished check. */ + const completeArgs = ( run: CiRunScope, key: string, name: string, - step: { id: string; name: string }, - tag: StepTag, + read: CheckResult, + ) => { + return { + run, + name, + ...identity(run, key), + conclusion: read.conclusion, + title: read.title, + summary: truncateSummary(read.summary ?? ""), + annotations: (read.annotations ?? []).map(normaliseAnnotation), + ...idFor(run, key), + }; + }; + + /** A job's check, or the pipeline's when there's no `job`. */ + const start = async ( + run: CiRunScope, + name: string, + job?: string, metadata?: () => Record, ) => { - const result = await githubStep(run, step, async () => { - await tagStep(run, tag, metadata?.()); + const key = job ?? "pipeline"; - const started = await sink.start({ run, name, ...identity(run, key) }); + const result = await githubStep( + run, + steps.startCheck(name, job), + async () => { + const started = await sink.start({ run, name, ...identity(run, key) }); - return { ...started, startedAt: Date.now() }; - }); + return { ...started, startedAt: Date.now() }; + }, + metadata, + ); if (result?.id) { checkRunIds.set(idKey(run, key), result.id); @@ -296,31 +320,23 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { const complete = async ( run: CiRunScope, - key: string, name: string, - step: { id: string; name: string }, result: () => CheckResult, - tag: StepTag, + job?: { path: string; conclusion: CheckConclusion }, metadata?: () => Record, ): Promise => { - const endedAt = await githubStep(run, step, async () => { - await tagStep(run, tag, metadata?.()); - - const read = result(); + const key = job?.path ?? "pipeline"; - await sink.complete({ - run, - name, - ...identity(run, key), - conclusion: read.conclusion, - title: read.title, - summary: truncateSummary(read.summary ?? ""), - annotations: (read.annotations ?? []).map(normaliseAnnotation), - ...idFor(run, key), - }); + const endedAt = await githubStep( + run, + steps.completeCheck(name, job), + async () => { + await sink.complete(completeArgs(run, key, name, result())); - return Date.now(); - }); + return Date.now(); + }, + metadata, + ); checkRunIds.delete(idKey(run, key)); @@ -333,17 +349,7 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return undefined; } - return start( - run, - "pipeline", - run.checkName, - { - id: `github › check:${run.checkName}:start`, - name: traceName.createCheck(run.checkName), - }, - { kind: "check" }, - metadata, - ); + return start(run, run.checkName, undefined, metadata); }, pipelineComplete: async ({ run, metadata, result }) => { @@ -351,18 +357,7 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return; } - await complete( - run, - "pipeline", - run.checkName, - { - id: `github › check:${run.checkName}:complete`, - name: traceName.completeCheck(run.checkName), - }, - result, - { kind: "check" }, - metadata, - ); + await complete(run, run.checkName, result, undefined, metadata); }, jobStart: async ({ run, jobPath, name }) => { @@ -370,16 +365,7 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return undefined; } - return start( - run, - jobPath, - jobCheckName(run, jobPath, name), - { - id: `github › check:${jobPath}:start`, - name: traceName.report(jobPath, "started"), - }, - { kind: "check", job: jobPath }, - ); + return start(run, jobCheckName(run, jobPath, name), jobPath); }, jobComplete: async ({ run, jobPath, name, ...result }) => { @@ -389,16 +375,11 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return complete( run, - jobPath, jobCheckName(run, jobPath, name), - { - id: `github › check:${jobPath}:complete`, - name: traceName.report(jobPath, traceName.outcome(result.conclusion)), - }, () => { return result; }, - { kind: "check", job: jobPath }, + { path: jobPath, conclusion: result.conclusion }, ); }, @@ -413,31 +394,21 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return closing(read()); } - const step = { - id: "github › check:jobs:complete", - name: traceName.report("jobs", "ended with the run"), - }; - - const closed = await githubStep(run, step, async () => { - await tagStep(run, { kind: "check" }); + const closed = await githubStep( + run, + steps.completeJobChecks(), + async () => { + const jobs = read(); - const jobs = read(); + for (const { jobPath, name, ...result } of jobs) { + const check = jobCheckName(run, jobPath, name); - for (const { jobPath, name, ...result } of jobs) { - await sink.complete({ - run, - name: jobCheckName(run, jobPath, name), - ...identity(run, jobPath), - conclusion: result.conclusion, - title: result.title, - summary: truncateSummary(result.summary ?? ""), - annotations: (result.annotations ?? []).map(normaliseAnnotation), - ...idFor(run, jobPath), - }); - } + await sink.complete(completeArgs(run, jobPath, check, result)); + } - return closing(jobs); - }); + return closing(jobs); + }, + ); // Outside the step, so a replay also forgets the IDs its memoized start // steps put back. @@ -458,22 +429,14 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { jobPath === undefined ? run.checkName : jobCheckName(run, jobPath, name); - const step = { - id: `github › check:${key}:retry:${run.attempt}`, - name: traceName.report( - jobPath ?? run.checkName, - traceName.retrying(run.attempt + 2, run.maxAttempts), - ), - }; + const step = steps.retryCheck( + checkName, + run.attempt, + run.maxAttempts, + jobPath, + ); await githubStep(run, step, async () => { - await tagStep( - run, - jobPath === undefined - ? { kind: "check" } - : { kind: "check", job: jobPath }, - ); - await sink.update?.({ run, name: checkName, @@ -490,17 +453,9 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { return; } - const step = { - id: `github › check:jobs:retry:${run.attempt}`, - name: traceName.report( - "jobs", - traceName.retrying(run.attempt + 2, run.maxAttempts), - ), - }; + const step = steps.retryJobChecks(run.attempt, run.maxAttempts); await githubStep(run, step, async () => { - await tagStep(run, { kind: "check" }); - for (const { jobPath, name } of read()) { await sink.update?.({ run, @@ -521,23 +476,18 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { const message = errorMessage(error).split("\n")[0]; - await githubStep( - run, - { - id: `github › check:${jobPath}:attempt:${attempt}`, - name: traceName.report(jobPath, traceName.retrying(attempt + 1, of)), - }, - async () => { - await sink.update?.({ - run, - name: jobCheckName(run, jobPath), - title: `Attempt ${attempt} of ${of}: ${message}`, - ...idFor(run, jobPath), - }); + const step = steps.failedAttempt(jobPath, attempt, of); - return null; - }, - ); + await githubStep(run, step, async () => { + await sink.update?.({ + run, + name: jobCheckName(run, jobPath), + title: `Attempt ${attempt} of ${of}: ${message}`, + ...idFor(run, jobPath), + }); + + return null; + }); }, target: ({ run, jobPath, name }) => { @@ -559,10 +509,8 @@ export const createCheckReporter = (sink: CheckSink): CheckReporter => { } const { parent } = run.build; - const step = { - id: `github › check:${parent.jobPath}:building`, - name: traceName.report(parent.jobPath, "building"), - }; + + const step = steps.buildingCheck(parent.jobPath); await githubStep(run, step, async () => { await sink.update?.({ diff --git a/packages/ci/src/github/helpers.ts b/packages/ci/src/github/helpers.ts index 1e6abc3e9..0d3ed4190 100644 --- a/packages/ci/src/github/helpers.ts +++ b/packages/ci/src/github/helpers.ts @@ -8,7 +8,8 @@ import { RetryAfterError } from "inngest"; import { CiUsageError } from "../errors.ts"; import { durablePath } from "../pipeline/durable.ts"; -import { traceName } from "../pipeline/names.ts"; +import { ciRun } from "../pipeline/metadata.ts"; +import { steps, traceName } from "../pipeline/names.ts"; import type { CiRunScope } from "../pipeline/scope.ts"; import { countApi, @@ -40,7 +41,7 @@ const helperStep = async ( key: string, fn: () => Promise, /** The step's name, which is the helper's method name by default. */ - name = traceName.githubHelper(helper), + name?: string, ): Promise => { const run = requireRunScope(`github.${helper}`); @@ -54,7 +55,7 @@ const helperStep = async ( const id = `${job ? `${job.path} › ` : ""}github › ${helper}:${key}`; return inHelperSpan(run, () => { - return run.step.run({ id, name }, fn) as Promise; + return ciRun(run, steps.githubHelper(id, helper, name), fn); }); }; diff --git a/packages/ci/src/machine/admin.test.ts b/packages/ci/src/machine/admin.test.ts new file mode 100644 index 000000000..0f53433d8 --- /dev/null +++ b/packages/ci/src/machine/admin.test.ts @@ -0,0 +1,70 @@ +/** + * Tests of the Sandboxes API helpers cleanup shares: paged lists read as one + * stream, and snapshot deletes that tolerate what is already gone. + * + * @module + */ + +import type { Inngest } from "inngest"; +import { describe, expect, test, vi } from "vitest"; +import { deleteSnapshots, pages } from "./admin.ts"; + +const notFound = Object.assign(new Error("gone"), { + code: "sandbox_snapshot_not_found", +}); + +describe("pages", () => { + test("reads every page in order, passing the cursor on", async () => { + const list = vi.fn(async (cursor?: string) => { + return cursor === "next" + ? { items: [3], page: { hasMore: false } } + : { items: [1, 2], page: { hasMore: true, cursor: "next" } }; + }); + + const read: number[] = []; + + for await (const item of pages(list)) { + read.push(item); + } + + expect(read).toEqual([1, 2, 3]); + expect(list.mock.calls).toEqual([[undefined], ["next"]]); + }); +}); + +describe("deleteSnapshots", () => { + test("counts what it deletes, skips what is gone, and reports what fails", async () => { + const deleted: string[] = []; + + const get = async (id: string) => { + if (id === "gone") { + throw notFound; + } + + if (id === "stuck") { + throw new Error("refused"); + } + + return { + delete: async () => { + deleted.push(id); + }, + }; + }; + + const client = { + sandboxes: { snapshots: { get } }, + } as unknown as Inngest.Any; + const onFailure = vi.fn(); + + const result = await deleteSnapshots( + client, + ["a", "gone", "stuck", "b"], + onFailure, + ); + + expect(result).toEqual({ deleted: ["a", "b"], failed: ["stuck"] }); + expect(deleted).toEqual(["a", "b"]); + expect(onFailure).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/ci/src/machine/admin.ts b/packages/ci/src/machine/admin.ts new file mode 100644 index 000000000..c5dc75129 --- /dev/null +++ b/packages/ci/src/machine/admin.ts @@ -0,0 +1,75 @@ +/** + * Listing and deleting what CI's cleanup finds in the Sandboxes API: paged + * lists read as one stream, and snapshot deletes that tolerate what is already + * gone. Cleanup steps, run cleanup and invalidation all go through these + * rather than writing the paging and not-found handling again. + * + * @module + */ + +import type { Inngest } from "inngest"; +import { isSnapshotNotFound } from "../util.ts"; + +/** A page of a list the Sandboxes API returns. */ +export interface Page { + items: T[]; + page: { hasMore: boolean; cursor?: string }; +} + +/** Every item of a paged list, in order, fetching the next page as it is read. */ +export async function* pages( + list: (cursor?: string) => Promise>, +): AsyncGenerator { + let cursor: string | undefined; + + do { + const { items, page } = await list(cursor); + + yield* items; + + cursor = page.hasMore ? page.cursor : undefined; + } while (cursor); +} + +/** Every sandbox, whoever made it. */ +export const listSandboxes = (client: Inngest.Any) => { + return pages((cursor) => { + return client.sandboxes.list({ ...(cursor ? { cursor } : {}), limit: 100 }); + }); +}; + +/** + * Delete snapshots by ID. One that is already gone counts for nothing, and one + * that can't be deleted is reported to `onFailure` and left to expire, so a + * snapshot that stays never stops the rest. + */ +export const deleteSnapshots = async ( + client: Inngest.Any, + ids: Iterable, + onFailure?: (id: string, error: unknown) => void, +): Promise<{ deleted: string[]; failed: string[] }> => { + const deleted: string[] = []; + const failed: string[] = []; + + for (const id of ids) { + try { + const snapshot = await client.sandboxes.snapshots.get(id); + + if (snapshot) { + await snapshot.delete(); + + deleted.push(id); + } + } catch (error) { + if (isSnapshotNotFound(error)) { + continue; + } + + failed.push(id); + + onFailure?.(id, error); + } + } + + return { deleted, failed }; +}; diff --git a/packages/ci/src/machine/command.ts b/packages/ci/src/machine/command.ts index 7cf420bee..e8579835e 100644 --- a/packages/ci/src/machine/command.ts +++ b/packages/ci/src/machine/command.ts @@ -13,7 +13,8 @@ import { CommandFailedError, CommandTimeoutError, } from "../errors.ts"; -import { ciStep, spans, traceName } from "../pipeline/names.ts"; +import { ciRun } from "../pipeline/metadata.ts"; +import { ciStep, spans, steps, traceName } from "../pipeline/names.ts"; import type { CiJobScope, MachineHandle } from "../pipeline/scope.ts"; import { countApi, @@ -601,12 +602,9 @@ const recordFailure = async ( exitCode: number, ): Promise => { try { - await scope.run.step.run( - subStep(stepId, "exit", traceName.exited(exitCode)), - () => { - throw new NonRetriableError(`exit ${exitCode}`); - }, - ); + await ciRun(scope.run, steps.recordCommandFailure(stepId, exitCode), () => { + throw new NonRetriableError(`exit ${exitCode}`); + }); } catch (error) { if (!(error instanceof StepError)) { throw error; diff --git a/packages/ci/src/machine/machine.ts b/packages/ci/src/machine/machine.ts index 9a36a46f1..0249a24d7 100644 --- a/packages/ci/src/machine/machine.ts +++ b/packages/ci/src/machine/machine.ts @@ -9,7 +9,8 @@ import type { Inngest } from "inngest"; import type { CachedSnapshot, CacheTarget } from "../cache/cache.ts"; import { resolveTakenName, snapshotState } from "../cache/cache.ts"; import { CiUsageError } from "../errors.ts"; -import { ciStep, spans, traceName } from "../pipeline/names.ts"; +import { ciRun } from "../pipeline/metadata.ts"; +import { spans, steps, traceName } from "../pipeline/names.ts"; import type { CiJobScope, CiRunScope, @@ -25,13 +26,8 @@ import { } from "../pipeline/scope.ts"; import { inSpan } from "../pipeline/spans.ts"; import type { MachineConfig } from "../types.ts"; -import { - boundedName, - errorMessage, - isSandboxNotFound, - isSnapshotNotFound, - slug, -} from "../util.ts"; +import { boundedName, errorMessage, isSandboxNotFound, slug } from "../util.ts"; +import { deleteSnapshots, listSandboxes } from "./admin.ts"; /** * Create this scope's machine if it doesn't have one yet. @@ -359,35 +355,22 @@ const discardFailedStart = async ( ?.sandboxId; try { - await run.step.run( - { - id: joinId(stepId, "discard"), - name: traceName.discardMachine, - }, + await ciRun( + run, + steps.discardMachine(stepId), async (): Promise<{ id?: string }> => { // Errors are swallowed inside the step so it never retries. let id = knownId; try { - let cursor: string | undefined; - - while (!id) { - const page = await run.ci.client.sandboxes.list({ - ...(cursor ? { cursor } : {}), - limit: 100, - }); - - id = page.items.find( - (sandbox: { name: string; status: string }) => { - return sandbox.name === name && sandbox.status !== "TERMINATED"; - }, - )?.id; - - if (id || !page.page.hasMore) { - break; - } + if (!id) { + for await (const sandbox of listSandboxes(run.ci.client)) { + if (sandbox.name === name && sandbox.status !== "TERMINATED") { + id = sandbox.id; - cursor = page.page.cursor; + break; + } + } } if (id) { @@ -738,15 +721,9 @@ export const destroyRunMachines = async ( run: CiRunScope, attempt = 0, ): Promise => { - await run.step.run( - ciStep( - `pipeline${scopeSeparator}cleanup${attempt > 0 ? ` (attempt ${attempt})` : ""}`, - traceName.cleanUpMachines, - ), - async () => { - return destroyOrphans(run.ci.client, run.runId); - }, - ); + await ciRun(run, steps.cleanUpMachines(attempt), () => { + return destroyOrphans(run.ci.client, run.runId); + }); }; /** @@ -771,41 +748,14 @@ export const deleteRunSnapshots = async ( // Always there, and reads the set when it runs, for the reason cleaning up // machines is. - await run.step.run( - ciStep( - `pipeline${scopeSeparator}cleanup:snapshots${attempt > 0 ? ` (attempt ${attempt})` : ""}`, - traceName.cleanUpSnapshots, - ), - async () => { - const deleted: string[] = []; - const failed: string[] = []; - - for (const id of [...run.createdSnapshots]) { - try { - const snapshot = await run.ci.client.sandboxes.snapshots.get(id); - - if (snapshot) { - await snapshot.delete(); - - deleted.push(id); - } - } catch (error) { - if (isSnapshotNotFound(error)) { - continue; - } - - failed.push(id); - - run.ci.logger?.warn( - { snapshotId: id, error }, - "Couldn't delete a snapshot this run took; it will expire on its own", - ); - } - } - - return { deleted, failed }; - }, - ); + await ciRun(run, steps.cleanUpSnapshots(attempt), () => { + return deleteSnapshots(run.ci.client, run.createdSnapshots, (id, error) => { + run.ci.logger?.warn( + { snapshotId: id, error }, + "Couldn't delete a snapshot this run took; it will expire on its own", + ); + }); + }); }; /** @@ -818,32 +768,22 @@ export const destroyOrphans = async ( runId: string, ): Promise<{ destroyed: number }> => { const prefix = `ci-${runId}-`; - let cursor: string | undefined; let destroyed = 0; - do { - const page = await client.sandboxes.list({ - ...(cursor ? { cursor } : {}), - limit: 100, - }); - - for (const sandbox of page.items) { - if (sandbox.name.startsWith(prefix)) { - try { - await sandbox.destroy(); + for await (const sandbox of listSandboxes(client)) { + if (sandbox.name.startsWith(prefix)) { + try { + await sandbox.destroy(); - destroyed++; - } catch (error) { - // Anything but "not found" fails the step so it retries. - if (!isSandboxNotFound(error)) { - throw error; - } + destroyed++; + } catch (error) { + // Anything but "not found" fails the step so it retries. + if (!isSandboxNotFound(error)) { + throw error; } } } - - cursor = page.page.hasMore ? page.page.cursor : undefined; - } while (cursor); + } return { destroyed }; }; diff --git a/packages/ci/src/pipeline/durable.ts b/packages/ci/src/pipeline/durable.ts index 757ef95fa..7d4b08032 100644 --- a/packages/ci/src/pipeline/durable.ts +++ b/packages/ci/src/pipeline/durable.ts @@ -7,6 +7,7 @@ import { getAsyncCtx } from "inngest/experimental"; import { CiUsageError } from "../errors.ts"; +import { stepOptions, steps } from "./names.ts"; import { getJobScope, getRunScope, nextStepId } from "./scope.ts"; export type DurableBehaviour = "step" | "direct"; @@ -179,7 +180,10 @@ const call = async ( const step = getRunScope()?.step ?? execution.ctx.step; // Named by the method called, without the job path or count in its ID. - return step.run({ id, name: overrides.name ?? label }, invoke); + return step.run( + stepOptions(steps.githubCall(id, label, overrides.name)), + invoke, + ); }; const invokeOnClient = async ( diff --git a/packages/ci/src/pipeline/job.ts b/packages/ci/src/pipeline/job.ts index e482b6d8d..18acc8bb3 100644 --- a/packages/ci/src/pipeline/job.ts +++ b/packages/ci/src/pipeline/job.ts @@ -32,8 +32,8 @@ import { snapshotMachine } from "../machine/machine.ts"; import type { AnyJob, CheckConclusion, JobConfig } from "../types.ts"; import { errorMessage, formatDuration, shortReason } from "../util.ts"; import type { CacheBuildData, CacheBuildResult } from "./cacheBuild.ts"; -import { tagStep } from "./metadata.ts"; -import { ciStep, traceName } from "./names.ts"; +import { ciRun } from "./metadata.ts"; +import { ciStep, steps, traceName } from "./names.ts"; import type { CiJobScope, CiRunScope } from "./scope.ts"; import { getRunScope, @@ -545,12 +545,7 @@ const jobSteps = async ({ const checkStartedAt = checked ? await checks.jobStart(target) : undefined; const startedAt = checkStartedAt ?? - (await durableNow( - run, - `start:${scope.path}`, - traceName.recordStartTime, - scope.path, - )); + (await durableNow(run, `start:${scope.path}`, scope.path, "started")); if (checked) { run.openChecks.set(scope.path, checkName); @@ -629,12 +624,7 @@ const jobSteps = async ({ const endedAt = checkEndedAt ?? - (await durableNow( - run, - `end:${scope.path}`, - traceName.recordEndTime, - scope.path, - )); + (await durableNow(run, `end:${scope.path}`, scope.path, "ended")); const durationMs = endedAt - startedAt; run.summaries.push({ @@ -679,12 +669,7 @@ const jobSteps = async ({ const endedAt = checkEndedAt ?? - (await durableNow( - run, - `end:${scope.path}`, - traceName.recordEndTime, - scope.path, - )); + (await durableNow(run, `end:${scope.path}`, scope.path, "ended")); run.summaries.push({ path: scope.path, @@ -705,12 +690,10 @@ const jobSteps = async ({ const durableNow = ( run: CiRunScope, id: string, - name: string, jobPath: string, + edge: "started" | "ended", ): Promise => { - return run.step.run(ciStep(id, name), async () => { - await tagStep(run, { kind: "job", job: jobPath }); - + return ciRun(run, steps.recordTime(id, jobPath, edge), () => { return Date.now(); }); }; diff --git a/packages/ci/src/pipeline/metadata.ts b/packages/ci/src/pipeline/metadata.ts index 72f2c1657..7672e3284 100644 --- a/packages/ci/src/pipeline/metadata.ts +++ b/packages/ci/src/pipeline/metadata.ts @@ -11,7 +11,7 @@ import { getAsyncCtx } from "inngest/experimental"; import type { CheckConclusion } from "../types.ts"; import { version } from "../version.ts"; -import { ciStep, traceName } from "./names.ts"; +import { type StepSpec, stepOptions, steps } from "./names.ts"; import type { CiRunScope } from "./scope.ts"; import { apiNames } from "./scope.ts"; @@ -141,6 +141,25 @@ export const tagStep = async ( } }; +/** + * Run `fn` as one of CI's own steps, from the `steps` catalog. The step is + * tagged with what its spec says it is for, and with `runValues` on the run. + */ +export const ciRun = ( + run: CiRunScope, + spec: StepSpec, + fn: () => T | Promise, + runValues?: () => Record, +): Promise => { + return run.step.run(stepOptions(spec), async () => { + if (spec.tag || runValues) { + await tagStep(run, spec.tag, runValues?.()); + } + + return fn(); + }) as Promise; +}; + /** * A step of its own that only carries run metadata and returns the time, for * a pipeline whose check is off and so has no step to ride on. @@ -150,9 +169,12 @@ export const metadataStep = ( id: string, runValues: () => Record, ): Promise => { - return run.step.run(ciStep(id, traceName.recordRunDetails), async () => { - await tagStep(run, undefined, runValues()); - - return Date.now(); - }); + return ciRun( + run, + steps.recordRunDetails(id), + () => { + return Date.now(); + }, + runValues, + ); }; diff --git a/packages/ci/src/pipeline/names.ts b/packages/ci/src/pipeline/names.ts index 475a259b5..a58ab1868 100644 --- a/packages/ci/src/pipeline/names.ts +++ b/packages/ci/src/pipeline/names.ts @@ -13,6 +13,7 @@ import type { CheckConclusion } from "../types.ts"; import { formatDuration } from "../util.ts"; import { version } from "../version.ts"; +import type { StepTag } from "./metadata.ts"; import type { CiRunScope } from "./scope.ts"; import { joinId, scopeSeparator } from "./scope.ts"; import type { SpanInfo, SpanKind } from "./spans.ts"; @@ -94,6 +95,21 @@ export const spans = { }, }; +/** What one of CI's own steps is: its ID, name, and what it's marked with. */ +export interface StepSpec { + /** The step's ID, which replays depend on. */ + id: string; + /** What the step is called in the trace. */ + name: string; + /** What the step is for, when it belongs to a job, a check or the cache. */ + tag?: StepTag; + /** + * Set for a step that CI's origin shouldn't mark: something you called, or a + * step inside a span that already carries it. + */ + unmarked?: true; +} + /** * A step CI runs for you where no span of CI's marks it, such as a command's * `Start process` inside the command's span, which is yours. @@ -102,6 +118,13 @@ export const ciStep = (id: string, name: string) => { return { id, name, ...originOption(ciOrigin) }; }; +/** The step options for a spec, marked as CI's work unless it says not. */ +export const stepOptions = (spec: StepSpec) => { + return spec.unmarked + ? { id: spec.id, name: spec.name } + : ciStep(spec.id, spec.name); +}; + /** How a finished check reads, as in `Report test: passed`. */ const outcomes: Partial> = { success: "passed", @@ -109,6 +132,233 @@ const outcomes: Partial> = { timed_out: "timed out", }; +/** A check update's name, as in `Report test: started`. */ +const report = (subject: string, status: string): string => { + return `Report ${subject}: ${status}`; +}; + +/** A retry's name, as in `retrying (attempt 2 of 3)`. */ +const retrying = (attempt: number, of: number): string => { + return `retrying (attempt ${attempt} of ${of})`; +}; + +const cacheTag = (job: string): StepTag => { + return { kind: "cache", job }; +}; + +const checkTag = (job?: string): StepTag => { + return job === undefined ? { kind: "check" } : { kind: "check", job }; +}; + +/** The ID of a step of a cleanup, which a retried cleanup repeats. */ +const cleanupId = (what: string, attempt: number): string => { + return joinId( + "pipeline", + `${what}${attempt > 0 ? ` (attempt ${attempt})` : ""}`, + ); +}; + +/** The rest of a spec, once its ID and name are known. */ +type StepExtras = Omit; + +/** A step with a fixed name, whose ID the caller makes. */ +const named = (name: string, extras: StepExtras = {}) => { + return (id: string): StepSpec => { + return { id, name, ...extras }; + }; +}; + +/** A step with a fixed ID and name. */ +const fixed = (id: string, name: string, extras: StepExtras = {}) => { + return (): StepSpec => { + return { id, name, ...extras }; + }; +}; + +/** + * Every step CI runs for you: its ID, name and tag, in one place. Adding a + * step or renaming one is an edit to its entry here, and the step itself is + * `ciRun(run, steps.x(...), fn)`. Steps that CI's origin shouldn't mark are + * `unmarked`. + */ +export const steps = { + // The cache. + cacheKey: (jobPath: string): StepSpec => { + return { + id: joinId(jobPath, "cache:key"), + name: "Check cache", + tag: cacheTag(jobPath), + }; + }, + + lookUpCache: (id: string, jobPath: string): StepSpec => { + return { id, name: "Look up cache", tag: cacheTag(jobPath) }; + }, + + resolveCacheName: named("Resolve cache name"), + checkSnapshotState: named("Check snapshot state"), + deleteBadSnapshot: named("Delete bad snapshot"), + + // Sandboxes. + discardMachine: (stepId: string): StepSpec => { + return { + id: joinId(stepId, "discard"), + name: "Discard sandbox", + unmarked: true, + }; + }, + + recordCommandFailure: (stepId: string, exitCode: number): StepSpec => { + return { + id: joinId(stepId, "exit"), + name: `Exited with code ${exitCode}`, + }; + }, + + cleanUpMachines: (attempt: number): StepSpec => { + return { id: cleanupId("cleanup", attempt), name: "Clean up sandboxes" }; + }, + + cleanUpEndedRun: fixed("destroy-orphans", "Clean up sandboxes"), + + cleanUpSnapshots: (attempt: number): StepSpec => { + return { + id: cleanupId("cleanup:snapshots", attempt), + name: "Clean up snapshots", + }; + }, + + // Checking out. + uploadWorkingTree: named("Upload working tree", { unmarked: true }), + cloneRepository: named("Clone repository", { unmarked: true }), + findChangedFiles: fixed("changed", "Find changed files", { unmarked: true }), + + // What you called on a run. + githubHelper: (id: string, helper: string, name = `github.${helper}`) => { + return { id, name, unmarked: true } satisfies StepSpec; + }, + + githubCall: (id: string, label: string, name = label) => { + return { id, name, unmarked: true } satisfies StepSpec; + }, + + addSummary: named("Add summary", { unmarked: true }), + addAnnotations: named("Add annotations", { unmarked: true }), + + // Reading from GitHub. + resolveRepository: (): StepSpec => { + return { + id: joinId("github", "repo:resolve"), + name: "Resolve repository", + unmarked: true, + }; + }, + + resolvePullRequest: (): StepSpec => { + return { + id: joinId("github", "pr:resolve"), + name: "pr:resolve", + unmarked: true, + }; + }, + + commentNotAllowed: (): StepSpec => { + return { + id: joinId("github", "comment:denied"), + name: "Comment: not allowed", + unmarked: true, + }; + }, + + // Checks. + startCheck: (check: string, job?: string): StepSpec => { + return { + id: joinId("github", `check:${job ?? check}:start`), + name: + job === undefined ? `Create check: ${check}` : report(job, "started"), + tag: checkTag(job), + unmarked: true, + }; + }, + + completeCheck: ( + check: string, + job?: { path: string; conclusion: CheckConclusion }, + ): StepSpec => { + return { + id: joinId("github", `check:${job?.path ?? check}:complete`), + name: job + ? report(job.path, outcomes[job.conclusion] ?? job.conclusion) + : `Complete check: ${check}`, + tag: checkTag(job?.path), + unmarked: true, + }; + }, + + completeJobChecks: (): StepSpec => { + return { + id: joinId("github", "check:jobs:complete"), + name: report("jobs", "ended with the run"), + tag: checkTag(), + unmarked: true, + }; + }, + + retryCheck: ( + check: string, + attempt: number, + of: number, + job?: string, + ): StepSpec => { + return { + id: joinId("github", `check:${job ?? "pipeline"}:retry:${attempt}`), + name: report(job ?? check, retrying(attempt + 2, of)), + tag: checkTag(job), + unmarked: true, + }; + }, + + retryJobChecks: (attempt: number, of: number): StepSpec => { + return { + id: joinId("github", `check:jobs:retry:${attempt}`), + name: report("jobs", retrying(attempt + 2, of)), + tag: checkTag(), + unmarked: true, + }; + }, + + failedAttempt: (job: string, attempt: number, of: number): StepSpec => { + return { + id: joinId("github", `check:${job}:attempt:${attempt}`), + name: report(job, retrying(attempt + 1, of)), + unmarked: true, + }; + }, + + buildingCheck: (job: string): StepSpec => { + return { + id: joinId("github", `check:${job}:building`), + name: report(job, "building"), + unmarked: true, + }; + }, + + // Times and run details, for steps that carry nothing else. + recordTime: ( + id: string, + job: string, + edge: "started" | "ended", + ): StepSpec => { + return { + id, + name: edge === "started" ? "Record start time" : "Record end time", + tag: { kind: "job", job }, + }; + }, + + recordRunDetails: named("Record run details"), +}; + export const traceName = { /** A job's span: its `name`, or its path when it has none. */ job: (run: CiRunScope, path: string): string => { @@ -138,40 +388,10 @@ export const traceName = { return `Wait ${formatDuration(ms)}`; }, - exited: (exitCode: number): string => { - return `Exited with code ${exitCode}`; - }, - buildInOwnRun: (path: string): string => { return `Build ${path} in its own run`; }, - createCheck: (check: string): string => { - return `Create check: ${check}`; - }, - - completeCheck: (check: string): string => { - return `Complete check: ${check}`; - }, - - /** A check update, as in `Report test: started`. */ - report: (subject: string, status: string): string => { - return `Report ${subject}: ${status}`; - }, - - retrying: (attempt: number, of: number): string => { - return `retrying (attempt ${attempt} of ${of})`; - }, - - outcome: (conclusion: CheckConclusion): string => { - return outcomes[conclusion] ?? conclusion; - }, - - /** A `github.*` helper you called, by its method name. */ - githubHelper: (helper: string): string => { - return `github.${helper}`; - }, - canUser: (login: string, permission: string): string => { return `Check ${login} can ${permission}`; }, @@ -185,19 +405,14 @@ export const traceName = { }, github: "GitHub", - resolveRepository: "Resolve repository", - commentNotAllowed: "Comment: not allowed", createMachine: "Create sandbox", restartMachine: "Restart sandbox", createFreshMachine: "Create sandbox (fresh)", retryCreate: "Retry create", - discardMachine: "Discard sandbox", prepareWorkspace: "Prepare workspace", saveMachine: "Save sandbox", snapshotMachine: "Snapshot sandbox", - cleanUpMachines: "Clean up sandboxes", - cleanUpSnapshots: "Clean up snapshots", /** A captured command's one step: it runs and returns its output at once. */ runAndReadOutput: "Run and read output", @@ -207,20 +422,4 @@ export const traceName = { stopProcess: "Stop process", stopAfterTimeout: "Stop after timeout", findStartedProcess: "Find started process", - - checkCache: "Check cache", - lookUpCache: "Look up cache", - resolveCacheName: "Resolve cache name", - deleteBadSnapshot: "Delete bad snapshot", - checkSnapshotState: "Check snapshot state", - - cloneRepository: "Clone repository", - uploadWorkingTree: "Upload working tree", - findChangedFiles: "Find changed files", - - recordStartTime: "Record start time", - recordEndTime: "Record end time", - recordRunDetails: "Record run details", - addSummary: "Add summary", - addAnnotations: "Add annotations", }; diff --git a/packages/ci/src/pipeline/pipeline.ts b/packages/ci/src/pipeline/pipeline.ts index 4b5c9ce63..8c8eb2c9e 100644 --- a/packages/ci/src/pipeline/pipeline.ts +++ b/packages/ci/src/pipeline/pipeline.ts @@ -48,8 +48,13 @@ import { formatDuration } from "../util.ts"; import type { CacheBuildData } from "./cacheBuild.ts"; import type { RegisteredJob } from "./job.ts"; import { conclusionForError, runJob } from "./job.ts"; -import { metadataStep, runEndMetadata, runStartMetadata } from "./metadata.ts"; -import { ciStep, traceName } from "./names.ts"; +import { + ciRun, + metadataStep, + runEndMetadata, + runStartMetadata, +} from "./metadata.ts"; +import { stepOptions, steps } from "./names.ts"; import type { CiInternals, CiRunScope } from "./scope.ts"; import { apiNames, @@ -569,8 +574,9 @@ const resolveConfiguredRepo = ( const { owner, name } = parseRepo(fullName); return inGitHubSpan(run, () => { - return run.step.run( - { id: `github › repo:resolve`, name: traceName.resolveRepository }, + return ciRun( + run, + steps.resolveRepository(), async (): Promise => { const base: RepoContext = { owner, name, fullName, sha: "" }; const provider = run.ci.github as GitHubProvider; @@ -601,7 +607,7 @@ const resolveConfiguredRepo = ( (run.event as { name?: string } | undefined)?.name ?? "manual", }; }, - ) as Promise; + ); }); }; @@ -616,8 +622,9 @@ const resolvePullRequestHead = ( run: CiRunScope, repo: RepoContext, ): Promise => { - return run.step.run( - { id: `github › pr:resolve`, name: "pr:resolve" }, + return ciRun( + run, + steps.resolvePullRequest(), async (): Promise => { const provider = run.ci.github as GitHubProvider; const number = repo.pullRequest?.number; @@ -649,7 +656,7 @@ const resolvePullRequestHead = ( }, }; }, - ) as Promise; + ); }; /** Errors that replaying the run would only reproduce. */ @@ -832,19 +839,16 @@ const checkCommentPermission = async ( if (!allowed) { await inGitHubSpan(run, () => { - return run.step.run( - { id: "github › comment:denied", name: traceName.commentNotAllowed }, - async () => { - const { stickyComment } = await import("../github/helpers.ts"); + return ciRun(run, steps.commentNotAllowed(), async () => { + const { stickyComment } = await import("../github/helpers.ts"); - await stickyComment( - "permission", - `@${login} you need \`${minPermission}\` permission to run \`${commentBody(run.event as { data?: unknown }).split(" ")[0]}\`.`, - ); + await stickyComment( + "permission", + `@${login} you need \`${minPermission}\` permission to run \`${commentBody(run.event as { data?: unknown }).split(" ")[0]}\`.`, + ); - return { denied: login }; - }, - ); + return { denied: login }; + }); }); } @@ -899,12 +903,9 @@ export const cleanupFunction = ({ async ({ event, step }: any) => { const runId = event?.data?.run_id ?? event?.data?.runId; - return step.run( - ciStep("destroy-orphans", traceName.cleanUpMachines), - async () => { - return runId ? destroyOrphans(client, runId) : { destroyed: 0 }; - }, - ); + return step.run(stepOptions(steps.cleanUpEndedRun()), () => { + return runId ? destroyOrphans(client, runId) : { destroyed: 0 }; + }); }, ); }; diff --git a/packages/ci/src/pipeline/stepNames.test.ts b/packages/ci/src/pipeline/stepNames.test.ts new file mode 100644 index 000000000..7c5dd178c --- /dev/null +++ b/packages/ci/src/pipeline/stepNames.test.ts @@ -0,0 +1,66 @@ +/** + * Tests that the step catalog keeps the IDs and names the trace and replays + * already depend on, and marks as CI's work only the steps that were marked. + * + * @module + */ + +import { describe, expect, test } from "vitest"; +import { ciOrigin, stepOptions, steps } from "./names.ts"; + +describe("the step catalog", () => { + test.each([ + [steps.cacheKey("test"), "test › cache:key", "Check cache"], + [steps.cleanUpMachines(0), "pipeline › cleanup", "Clean up sandboxes"], + [ + steps.cleanUpSnapshots(2), + "pipeline › cleanup:snapshots (attempt 2)", + "Clean up snapshots", + ], + [steps.cleanUpEndedRun(), "destroy-orphans", "Clean up sandboxes"], + [ + steps.recordCommandFailure("test › 1", 2), + "test › 1 › exit", + "Exited with code 2", + ], + [steps.startCheck("CI"), "github › check:CI:start", "Create check: CI"], + [ + steps.startCheck("CI / test", "test"), + "github › check:test:start", + "Report test: started", + ], + [ + steps.completeCheck("CI / test", { path: "test", conclusion: "failure" }), + "github › check:test:complete", + "Report test: failed", + ], + [ + steps.retryCheck("CI", 0, 3), + "github › check:pipeline:retry:0", + "Report CI: retrying (attempt 2 of 3)", + ], + [ + steps.failedAttempt("test", 1, 3), + "github › check:test:attempt:1", + "Report test: retrying (attempt 2 of 3)", + ], + [steps.githubHelper("id", "comment"), "id", "github.comment"], + [ + steps.recordTime("start:test", "test", "ended"), + "start:test", + "Record end time", + ], + ])("%#", (spec, id, name) => { + expect({ id: spec.id, name: spec.name }).toEqual({ id, name }); + }); + + test("marks CI's own work with its origin, and an unmarked step without", () => { + expect(stepOptions(steps.cacheKey("test"))).toMatchObject({ + "~origin": ciOrigin, + }); + expect(stepOptions(steps.findChangedFiles())).toEqual({ + id: "changed", + name: "Find changed files", + }); + }); +}); diff --git a/packages/ci/src/report.ts b/packages/ci/src/report.ts index c66c13230..d22675699 100644 --- a/packages/ci/src/report.ts +++ b/packages/ci/src/report.ts @@ -4,7 +4,8 @@ * @module */ -import { traceName } from "./pipeline/names.ts"; +import { ciRun } from "./pipeline/metadata.ts"; +import { steps } from "./pipeline/names.ts"; import { countApi, getJobScope, @@ -44,7 +45,7 @@ export const report = { const id = nextStepId(run, job?.path, "report:summary"); - await run.step.run({ id, name: traceName.addSummary }, () => { + await ciRun(run, steps.addSummary(id), () => { return { length: markdown.length, }; @@ -95,7 +96,7 @@ export const report = { const id = nextStepId(run, job?.path, "report:annotate"); - await run.step.run({ id, name: traceName.addAnnotations }, () => { + await ciRun(run, steps.addAnnotations(id), () => { return { count: valid.length, }; diff --git a/packages/ci/src/units.test.ts b/packages/ci/src/units.test.ts index c15e89fe0..d5aefad5a 100644 --- a/packages/ci/src/units.test.ts +++ b/packages/ci/src/units.test.ts @@ -6,7 +6,8 @@ */ import { describe, expect, test } from "vitest"; -import { cacheScopes, lookupCache, snapshotName } from "./cache/cache.ts"; +import { cacheScopes, lookupCache } from "./cache/cache.ts"; +import { formatName } from "./cache/names.ts"; import { CiUsageError } from "./errors.ts"; import { batchAnnotations, @@ -684,7 +685,12 @@ describe("formatting", () => { }); describe("looking a cached snapshot up by name", () => { - const name = snapshotName("global", "setup", "k1"); + const name = formatName({ + kind: "cache", + scope: "global", + jobId: "setup", + ownKey: "k1", + }); // One moment for every fixture, so two snapshots built apart still match. const now = Date.now(); @@ -805,12 +811,29 @@ describe("looking a cached snapshot up by name", () => { describe("snapshot names", () => { test("say what they are for", () => { - expect(snapshotName("pr:4", "setup", "abc")).toBe("ci/pr:4/setup/abc"); + expect( + formatName({ + kind: "cache", + scope: "pr:4", + jobId: "setup", + ownKey: "abc", + }), + ).toBe("ci/pr:4/setup/abc"); }); test("too long a name keeps its start and stays unique", () => { - const long = snapshotName("main", "j".repeat(300), "abc"); - const other = snapshotName("main", "j".repeat(300), "abd"); + const long = formatName({ + kind: "cache", + scope: "main", + jobId: "j".repeat(300), + ownKey: "abc", + }); + const other = formatName({ + kind: "cache", + scope: "main", + jobId: "j".repeat(300), + ownKey: "abd", + }); expect(long).toHaveLength(255); expect(long.startsWith("ci/main/jjj")).toBe(true); From ee37919c3aff2f2f9f62d08b5c51f625325dee1b Mon Sep 17 00:00:00 2001 From: x Date: Tue, 13 Oct 2026 17:15:45 +0000 Subject: [PATCH 06/18] feat(ci): check out another repository or ref with checkout({ repo, ref }) Adds repo and ref to checkout() and files(), resolved to a commit once per run, with the CheckoutSource list, once() and localRepo(). Co-Authored-By: Claude Opus 5.5 --- .changeset/ci-checkout-repo.md | 5 + packages/ci/README.md | 24 + packages/ci/src/cache/cache.ts | 145 ++++- packages/ci/src/checkout/checkout.ts | 168 ++++-- packages/ci/src/github/README.md | 1 + packages/ci/src/github/auth.ts | 98 +++- packages/ci/src/github/source.test.ts | 806 ++++++++++++++++++++++++++ packages/ci/src/github/source.ts | 326 +++++++++++ packages/ci/src/index.ts | 1 + packages/ci/src/pipeline/names.ts | 18 +- packages/ci/src/pipeline/pipeline.ts | 17 +- packages/ci/src/pipeline/scope.ts | 27 + packages/ci/src/testing/fakeGitHub.ts | 45 +- packages/ci/src/types.ts | 13 +- packages/ci/src/util.ts | 49 ++ 15 files changed, 1653 insertions(+), 90 deletions(-) create mode 100644 .changeset/ci-checkout-repo.md create mode 100644 packages/ci/src/github/source.test.ts create mode 100644 packages/ci/src/github/source.ts diff --git a/.changeset/ci-checkout-repo.md b/.changeset/ci-checkout-repo.md new file mode 100644 index 000000000..c4dba711b --- /dev/null +++ b/.changeset/ci-checkout-repo.md @@ -0,0 +1,5 @@ +--- +"@inngest/ci": minor +--- + +`checkout()` and `files()` take `repo` and `ref`, to read another repository or ref. The ref is resolved to a commit once per run, and the GitHub App installation is found from the repository. `checkout({ repo })` also works in a run whose trigger has no repository. diff --git a/packages/ci/README.md b/packages/ci/README.md index bddaf9da9..9cb58f522 100644 --- a/packages/ci/README.md +++ b/packages/ci/README.md @@ -51,6 +51,30 @@ const test = ci.job({ id: "test", from: base }, async () => { A pipeline run is one trace. `lint` and `test` start from a snapshot of `base`, and a failed command runs again without rerunning the jobs that passed. +## Another repository or ref + +`checkout()` and `files()` take `repo` and `ref`. A ref is a branch, tag or commit, and it's resolved to a commit once per run. + +```ts +await checkout({ repo: "acme/platform" }); // default branch +await checkout({ repo: "acme/platform", ref: "v2.1.0" }); +await checkout({ ref: "main" }); // this repository + +const image = ci.job( + { + id: "image", + cache: { + key: files("images/node-base/**", { repo: "acme/platform", ref: "main" }), + }, + }, + async () => { + await checkout({ repo: "acme/platform", ref: "main" }); + }, +); +``` + +A run whose trigger has no repository, like a cron, needs `repo`. The GitHub App must have access to the repository. Locally, `checkout()` still uploads your working tree unless you name another repository or a `ref`. + ## Install ```bash diff --git a/packages/ci/src/cache/cache.ts b/packages/ci/src/cache/cache.ts index 59f072bdd..bea1e23c7 100644 --- a/packages/ci/src/cache/cache.ts +++ b/packages/ci/src/cache/cache.ts @@ -10,19 +10,29 @@ * @module */ +import { NonRetriableError } from "inngest"; +import { mapGitHubError, rest } from "../github/rest.ts"; +import type { ResolvedSource } from "../github/source.ts"; +import { + octokitForSource, + resolveSource, + targetsRunRepo, +} from "../github/source.ts"; import { ciRun } from "../pipeline/metadata.ts"; import type { StepSpec } from "../pipeline/names.ts"; import { steps } from "../pipeline/names.ts"; import type { CiJobScope, CiRunScope } from "../pipeline/scope.ts"; -import { countApi, joinId, rootRunIdOf } from "../pipeline/scope.ts"; +import { countApi, joinId, localRepo, rootRunIdOf } from "../pipeline/scope.ts"; import type { CacheConfig, CacheKey, CacheKeyPart, + FilesOptions, JobConfig, RepoContext, } from "../types.ts"; import { + filterPaths, formatRelative, hash, isSnapshotNotFound, @@ -87,8 +97,9 @@ const sharedScopes = ( * Resolve the parts of a cache key into a single hash. * * `files()` parts are resolved against the repository: the git tree for the - * run's commit on GitHub, or the working tree locally, so a key changes - * exactly when the matched files do. + * run's commit on GitHub (or for the commit its `repo` and `ref` resolved to), + * or the working tree locally, so a key changes exactly when the matched files + * do. */ export const resolveCacheKey = async ( run: CiRunScope, @@ -102,47 +113,115 @@ export const resolveCacheKey = async ( return hash(await key()); } - const parts = Array.isArray(key) ? key : [key]; const resolved: string[] = []; - for (const part of parts) { - if (typeof part === "string") { - resolved.push(part); + for (const part of keyParts(key)) { + resolved.push( + typeof part === "string" ? part : await resolveFilesPart(run, part), + ); + } - continue; - } + return hash(resolved.join("\0")); +}; - resolved.push(await resolveFilesPart(run, part)); +/** The parts of a cache key, which a function key doesn't have. */ +const keyParts = (key: CacheKey | undefined): (CacheKeyPart | string)[] => { + if (key === undefined || typeof key === "function") { + return []; } - return hash(resolved.join("\0")); + return Array.isArray(key) ? key : [key]; +}; + +/** + * Resolve the repository and ref of every `files()` part that reads another + * commit than the run's own, each in a memoized step. A key is worked out + * inside a step, which can't start another, so this runs just before it. + */ +export const resolveKeySources = async ( + run: CiRunScope, + key: CacheKey | undefined, +): Promise => { + for (const part of keyParts(key)) { + if (typeof part !== "string" && !targetsRunRepo(run, part)) { + await resolveSource(run, part); + } + } }; const resolveFilesPart = async ( run: CiRunScope, part: CacheKeyPart, ): Promise => { - const repo = run.repo; + if (!targetsRunRepo(run, part)) { + return hashRemoteFiles(run, await resolveSource(run, part), part); + } + + const local = localRepo(run); - if (repo?.local && process.env.INNGEST_CI_GITHUB !== "live") { + if (local) { const { hashLocalFiles } = await import("./localCache.ts"); - return hashLocalFiles(repo.local.path, part.patterns); + return hashLocalFiles(local.path, part.patterns); } + const repo = run.repo; + if (!repo) { return `files:${part.patterns.join(",")}`; } - const { rest } = await import("../github/rest.ts"); - const { filterPaths } = await import("../util.ts"); - const tree = await rest.git.getTree({ tree_sha: repo.sha, recursive: "1", }); - const entries = (tree.tree ?? []) + return hashTree(tree, part.patterns, repo.fullName); +}; + +/** + * Hash the matched files of another repository or ref, at the commit it + * resolved to, with a client for the installation that can read it. + */ +const hashRemoteFiles = async ( + run: CiRunScope, + source: ResolvedSource, + part: CacheKeyPart, +): Promise => { + const octokit = await octokitForSource(run, source); + + try { + const { data } = await octokit.rest.git.getTree({ + owner: source.owner, + repo: source.name, + tree_sha: source.sha, + recursive: "1", + }); + + return hashTree(data, part.patterns, source.fullName); + } catch (error) { + throw mapGitHubError(error) ?? error; + } +}; + +/** A hash of the blobs in a git tree that match the patterns. */ +const hashTree = ( + listing: { + tree?: { type?: string; path?: string; sha?: string }[]; + truncated?: boolean; + }, + patterns: string[], + fullName: string, +): string => { + // A truncated listing would hash only some of the files, and a key that + // ignores the rest never changes when they do. + if (listing.truncated) { + throw new NonRetriableError( + `\`${fullName}\` is too large to hash \`files()\` from GitHub's tree API, which cut the file listing short. Use narrower patterns, or a string key.`, + ); + } + + const entries = (listing.tree ?? []) .filter((entry) => { return entry.type === "blob" && entry.path; }) @@ -154,7 +233,7 @@ const resolveFilesPart = async ( entries.map((entry) => { return entry.path; }), - { include: part.patterns }, + { include: patterns }, ); const byPath = new Map( @@ -270,6 +349,8 @@ export const cacheTarget = async ( countApi("cache"); + await resolveKeySources(run, cache.key); + const ownKey = await ciRun(run, steps.cacheKey(job.path), () => { return jobCacheKey(run, cache, input, base); }); @@ -710,14 +791,38 @@ export const describeCached = ( * Contents are read from the git tree for the run's commit, or from the * working tree locally, so uncommitted changes change the key too. * + * End with `{ repo, ref }` to read another repository or ref. `ref` is a + * branch, tag or commit, and `repo` is `"owner/name"` with `ref` defaulting to + * its default branch. The ref is resolved to a commit once per run, so a branch + * that moves changes the key exactly when the matched files change. Locally, + * this reads from GitHub too, unless it names the local repository with no + * `ref`. + * + * ```ts + * cache: { + * key: files("images/node-base/**", { repo: "acme/platform", ref: "main" }), + * } + * ``` + * * Patterns support `**`, `*`, `?`, and `{a,b}`. */ export const files = ( - /** Glob patterns for the files to hash. */ ...patterns: string[] + /** Glob patterns for the files to hash, then optionally where to read them from. */ + ...args: string[] | [...patterns: string[], options: FilesOptions] ): CacheKeyPart => { + const list: (string | FilesOptions)[] = args; + const last = list.at(-1); + const options = typeof last === "object" ? last : undefined; + + const patterns = list.filter((arg): arg is string => { + return typeof arg === "string"; + }); + return { kind: "inngest/ci.cacheKeyPart", type: "files", patterns, + ...(options?.repo ? { repo: options.repo } : {}), + ...(options?.ref ? { ref: options.ref } : {}), }; }; diff --git a/packages/ci/src/checkout/checkout.ts b/packages/ci/src/checkout/checkout.ts index e0c7198c1..718d8b9b3 100644 --- a/packages/ci/src/checkout/checkout.ts +++ b/packages/ci/src/checkout/checkout.ts @@ -6,14 +6,23 @@ */ import { CiUsageError } from "../errors.ts"; +import type { ResolvedSource } from "../github/source.ts"; +import { + resolveSource, + sameRepo, + targetsRunRepo, + tokenForSource, +} from "../github/source.ts"; import { ensureMachine } from "../machine/machine.ts"; import { ciRun } from "../pipeline/metadata.ts"; +import type { StepSpec } from "../pipeline/names.ts"; import { steps } from "../pipeline/names.ts"; import type { CiRunScope, MachineHandle } from "../pipeline/scope.ts"; import { countApi, defaultCwd, joinId, + localRepo, requireJobScope, } from "../pipeline/scope.ts"; import type { RepoContext } from "../types.ts"; @@ -24,7 +33,15 @@ import { buildWorkingTreeTarball } from "./tarball.ts"; const maxUploadBytes = 100 * 1024 * 1024; interface CheckoutOptions { - /** The commit or ref to check out. Defaults to the run's. */ + /** + * The repository to check out, as `owner/name`. Defaults to the run's. Needed + * when the run's trigger has no repository. + */ + repo?: string; + /** + * The branch, tag or commit to check out. Defaults to the run's commit, or to + * the default branch when `repo` is another repository. + */ ref?: string; /** Also initialise submodules, recursively. */ submodules?: boolean; @@ -61,52 +78,124 @@ interface CheckoutOptions { * }); * ``` * + * Name another repository, a branch, a tag or a commit with `repo` and `ref`. + * The ref is resolved to a commit once per run, so every job and every replay + * checks out the same one. Without `repo`, `ref` overrides the run's commit. A + * run whose trigger has no repository, like a cron, needs `repo`. + * + * ```ts + * await checkout({ repo: "acme/platform" }); // default branch + * await checkout({ repo: "acme/platform", ref: "v2.1.0" }); + * await checkout({ ref: "main" }); // this repository + * ``` + * + * Don't pass a `repo` taken from untrusted event data: it picks what gets + * cloned with your credentials. + * + * The GitHub App must be installed on the other repository's owner and have + * access to it. The installation is found from the repository, so it doesn't + * have to be the one that triggered the run. + * * Locally, it uploads your working tree, including uncommitted changes, so * there's nothing to push before running a pipeline. Over an existing local * checkout the files are extracted on top, so files deleted locally since can - * linger on the machine. Against GitHub, it clones - * the commit that triggered the run with a short-lived installation token that - * never leaves the step handler, so it's never in step input, step output, or - * the trace. + * linger on the machine. With another `repo` or an explicit `ref` it clones + * from GitHub, as CI does. Against GitHub, it clones the commit with a + * short-lived installation token that never leaves the step handler, so it's + * never in step input, step output, or the trace. * - * @throws {CiUsageError} When called outside a job, or when the run has no - * repository to check out. + * @throws {CiUsageError} When called outside a job, or when neither the run + * nor `repo` names a repository to check out. + * @throws {NonRetriableError} When the GitHub App can't access the repository, + * or the ref doesn't exist in it. */ export const checkout = async (opts: CheckoutOptions = {}): Promise => { const scope = requireJobScope("checkout"); countApi("checkout"); const { run } = scope; - const repo = run.repo; const target = opts.path ?? defaultCwd; + const id = joinId(scope.path, "checkout"); - const local = - repo?.local && process.env.INNGEST_CI_GITHUB !== "live" ? repo.local : null; + for (const source of checkoutSources) { + const plan = await source(run, opts, target, id); - if (!local && !repo) { - throw new CiUsageError( - "`checkout()` needs a repository. This run's trigger doesn't have one, so set `repo: \"owner/name\"` on the pipeline or send an event with repository data.", - ); + if (!plan) { + continue; + } + + const machine = await ensureMachine(scope); + + await ciRun(run, plan.spec, () => { + return plan.fill(machine); + }); + + scope.cwd ??= target; + + return; } - const machine = await ensureMachine(scope); - const stepId = joinId(scope.path, "checkout"); + throw new CiUsageError( + '`checkout()` needs a repository. This run\'s trigger doesn\'t have one, so pass `checkout({ repo: "owner/name" })` or set `repo: "owner/name"` on the pipeline.', + ); +}; - await ciRun( - run, - local ? steps.uploadWorkingTree(stepId) : steps.cloneRepository(stepId), - async () => { - if (local) { - return uploadWorkingTree(run, machine, local.path, target); - } +/** The checkout step, and how it fills the job's machine. */ +interface CheckoutPlan { + spec: StepSpec; + fill: (machine: MachineHandle) => Promise; +} - return cloneFromGithub(run, machine, repo as RepoContext, opts, target); +/** + * A place `checkout()` can get files from. It returns the plan for a checkout + * it serves and `undefined` for one it doesn't. It runs outside the step, so + * it's where anything a step can't do for itself, like resolving a ref, goes. + * + * A new source, such as a tarball URL, is one more entry in `checkoutSources`. + */ +type CheckoutSource = ( + run: CiRunScope, + opts: CheckoutOptions, + target: string, + id: string, +) => Promise; + +/** The working tree, for the run's own repository when it's run locally. */ +const localTree: CheckoutSource = async (run, opts, target, id) => { + const local = localRepo(run); + + // Another repository, or an explicit ref, is cloned even over a local run. + if (!local || !targetsRunRepo(run, opts)) { + return undefined; + } + + return { + spec: steps.uploadWorkingTree(id), + fill: (machine) => { + return uploadWorkingTree(run, machine, local.path, target); }, - ); + }; +}; + +/** A clone from GitHub, of the run's repository or another. */ +const githubClone: CheckoutSource = async (run, opts, target, id) => { + if (!opts.repo && !run.repo) { + return undefined; + } - scope.cwd ??= target; + const source = await resolveSource(run, opts); + + return { + spec: steps.cloneRepository(id), + fill: (machine) => { + return cloneFromGithub(run, machine, source, opts, target); + }, + }; }; +/** In order: the first that serves a checkout does. */ +const checkoutSources: CheckoutSource[] = [localTree, githubClone]; + const getSandbox = async (run: CiRunScope, machine: MachineHandle) => { const sandbox = await run.ci.client.sandboxes.get(machine.id); @@ -159,7 +248,7 @@ const uploadWorkingTree = async ( * already a git checkout it's updated to `sha` instead of cloned. */ export const cloneScript = (args: { - repo: RepoContext; + repo: Pick; opts: CheckoutOptions; target: string; sha: string; @@ -173,6 +262,7 @@ export const cloneScript = (args: { `\`checkout()\` was given the ref \`${sha}\`, which git would read as an option.`, ); } + const target = shellEscape(args.target); const filter = opts.history === "full" ? "" : "--filter=blob:none"; @@ -212,20 +302,30 @@ export const cloneScript = (args: { const cloneFromGithub = async ( run: CiRunScope, machine: MachineHandle, - repo: RepoContext, + source: ResolvedSource, opts: CheckoutOptions, target: string, ) => { // The token is minted here, inside the step handler, so it's never part of // the step's input or output. Both of those show in the trace. - const { token } = await import("../github/helpers.ts"); - const accessToken = await token(); + const accessToken = await tokenForSource(run, source); const sandbox = await getSandbox(run, machine); - const sha = opts.ref ?? repo.sha; - const url = `https://x-access-token:${accessToken}@github.com/${repo.fullName}.git`; - const script = cloneScript({ repo, opts, target, sha }); + const url = `https://x-access-token:${accessToken}@github.com/${source.fullName}.git`; + + // A fork's pull request head is only fetched for the run's own commit. + const own = + sameRepo(source.fullName, run.repo?.fullName) && + run.repo?.sha === source.sha; + const pullRequest = own ? run.repo?.pullRequest : undefined; + + const script = cloneScript({ + repo: { ...(pullRequest ? { pullRequest } : {}) }, + opts, + target, + sha: source.sha, + }); const result = await sandbox.commands.run(["/bin/sh", "-c", script], { environment: { CI_REPO_URL: url }, @@ -238,5 +338,5 @@ const cloneFromGithub = async ( ); } - return { sha, path: target, source: "github" as const }; + return { sha: source.sha, path: target, source: "github" as const }; }; diff --git a/packages/ci/src/github/README.md b/packages/ci/src/github/README.md index 113b1fb5f..9d4d85017 100644 --- a/packages/ci/src/github/README.md +++ b/packages/ci/src/github/README.md @@ -8,4 +8,5 @@ Everything GitHub. - `fixtures.ts`: local event fixtures standing in for webhooks when running from a working tree. - `checks.ts`: check run, commit status and console reporting. - `rest.ts`: the durable REST client. +- `source.ts`: resolving the repository and ref `checkout()` and `files()` read to a commit, and finding the installation that can read it. - `helpers.ts`: the `github.*` helpers, gathered in `index.ts`. diff --git a/packages/ci/src/github/auth.ts b/packages/ci/src/github/auth.ts index 1a4210a5e..e6c21a845 100644 --- a/packages/ci/src/github/auth.ts +++ b/packages/ci/src/github/auth.ts @@ -8,12 +8,19 @@ import { createAppAuth } from "@octokit/auth-app"; import { Octokit } from "@octokit/rest"; +import { NonRetriableError } from "inngest"; import { CiUsageError } from "../errors.ts"; +import { mapGitHubError } from "./rest.ts"; export interface AuthContext { installationId?: number; owner?: string; repo?: string; + /** + * Narrow the installation token to these repositories of the installation. + * Without it a token can reach everything the installation can. + */ + repositoryNames?: string[]; } /** @@ -29,6 +36,12 @@ export interface GitHubProvider { readonly reporter: "checks" | "statuses" | "console"; octokit(ctx?: AuthContext): Promise; token(ctx?: AuthContext): Promise; + /** + * The installation that can access a repository, or `undefined` when none + * can. Only a GitHub App has installations, so other providers leave this + * out and use the same credentials for every repository. + */ + installationFor?(owner: string, repo: string): Promise; } export interface GitHubAppProvider extends GitHubProvider { @@ -87,9 +100,28 @@ export const githubApp = ( return { appId, privateKey: privateKey.replace(/\\n/g, "\n") }; }; - const octokitFor = async (ctx?: AuthContext): Promise => { + /** Where and how every client here makes its calls. */ + const transport = { + ...(opts.baseUrl ? { baseUrl: opts.baseUrl } : {}), + ...(opts.fetch ? { request: { fetch: opts.fetch } } : {}), + }; + + const appOctokit = (installationId?: number): Octokit => { const { appId, privateKey } = resolve(); + return new Octokit({ + authStrategy: createAppAuth, + auth: { + appId, + privateKey, + ...(installationId ? { installationId } : {}), + }, + ...transport, + }); + }; + + /** A client for the installation, which can reach all of it. */ + const installationOctokit = (ctx?: AuthContext): Octokit => { const installationId = ctx?.installationId ?? (process.env.GITHUB_INSTALLATION_ID @@ -102,27 +134,67 @@ export const githubApp = ( ); } - return new Octokit({ - authStrategy: createAppAuth, - auth: { appId, privateKey, installationId }, - ...(opts.baseUrl ? { baseUrl: opts.baseUrl } : {}), - ...(opts.fetch ? { request: { fetch: opts.fetch } } : {}), - }); + return appOctokit(installationId); + }; + + /** A token for the installation, narrowed to `repositoryNames` when given. */ + const installationToken = async (ctx?: AuthContext): Promise => { + const auth = (await installationOctokit(ctx).auth({ + type: "installation", + ...(ctx?.repositoryNames ? { repositoryNames: ctx.repositoryNames } : {}), + })) as { token: string }; + + return auth.token; + }; + + const octokitFor = async (ctx?: AuthContext): Promise => { + if (!ctx?.repositoryNames) { + return installationOctokit(ctx); + } + + // A narrowed client authenticates with a token that reaches only those + // repositories. + return new Octokit({ auth: await installationToken(ctx), ...transport }); }; return { kind: "app", reporter: "checks", octokit: octokitFor, - token: async (ctx) => { - const octokit = await octokitFor(ctx); + installationFor: async (owner, repo) => { + try { + // Asked as the App itself, since an installation is what's being found. + const { data } = await appOctokit().rest.apps.getRepoInstallation({ + owner, + repo, + }); + + return data.id; + } catch (error) { + const status = (error as { status?: number }).status; + + if (status === 404) { + return undefined; + } + + const mapped = mapGitHubError(error); + + if (status === 401 || status === 403) { + // A rate limit is a 403 too, and retries. + if (mapped && !(mapped instanceof NonRetriableError)) { + throw mapped; + } - const auth = (await octokit.auth({ type: "installation" })) as { - token: string; - }; + throw new NonRetriableError( + "GitHub rejected the GitHub App's credentials. The App may be suspended, or `appId` and `privateKey` may be wrong.", + { cause: error }, + ); + } - return auth.token; + throw mapped ?? error; + } }, + token: installationToken, }; }; diff --git a/packages/ci/src/github/source.test.ts b/packages/ci/src/github/source.test.ts new file mode 100644 index 000000000..dc14ddb7e --- /dev/null +++ b/packages/ci/src/github/source.test.ts @@ -0,0 +1,806 @@ +/** + * Tests for `checkout()` and `files()` reading another repository or ref: the + * ref is resolved to a commit once, the commit is what's cloned and keyed, the + * installation is found for the repository, and the local working tree is only + * left for the run's own repository. + * + * @module + */ + +import { execFileSync } from "node:child_process"; +import { generateKeyPairSync } from "node:crypto"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, test } from "vitest"; +import { files } from "../cache/cache.ts"; +import { checkout } from "../checkout/checkout.ts"; +import { $ } from "../machine/command.ts"; +import { createCi } from "../pipeline/createCi.ts"; +import { createCiTestClient } from "../testing/client.ts"; +import { prEvent, prTrigger } from "../testing/events.ts"; +import { createFakeGitHub } from "../testing/fakeGitHub.ts"; +import { createFakeSandboxApi } from "../testing/fakeSandbox.ts"; +import { runFunction } from "../testing/runFunction.ts"; +import type { GitHubProvider } from "./auth.ts"; +import { githubApp, githubToken } from "./auth.ts"; + +type FakeGitHub = ReturnType; + +type Ci = ReturnType["ci"]; + +const secret = "ghs_never_in_a_trace"; + +const nightly = { name: "test/nightly", data: {} }; + +/** A CI client whose GitHub calls go to a fake, with the given provider. */ +const setup = ( + provider: (gh: FakeGitHub) => GitHubProvider = (gh) => { + return githubToken({ + token: secret, + baseUrl: "https://api.github.test", + fetch: gh.fetch, + }); + }, +) => { + const api = createFakeSandboxApi(); + const gh = createFakeGitHub(); + + const ci = createCi(createCiTestClient(api), { + github: { ...provider(gh), reporter: "statuses" }, + runUrl: ({ runId }) => { + return `http://localhost:8288/run?runID=${runId}`; + }, + }); + + return { api, gh, ci }; +}; + +/** Make `acme/platform`'s `main` branch resolve to a commit. */ +const platform = (gh: FakeGitHub, sha = "cafe1234") => { + gh.route("GET /repos/acme/platform", { default_branch: "main" }); + + gh.route("GET /repos/acme/platform/commits/main", { sha }); + + gh.route("GET /repos/acme/platform/commits/v2", { sha: "beef5678" }); +}; + +/** Make `ref` of the run's own repository resolve to a commit. */ +const ownRepo = (gh: FakeGitHub, ref: string, sha: string) => { + gh.route("GET /repos/inngest/inngest-js", { default_branch: "main" }); + + gh.route(`GET /repos/inngest/inngest-js/commits/${ref}`, { sha }); +}; + +const cloneScripts = (api: ReturnType) => { + return api.commands + .map((argv) => { + return argv[2] ?? ""; + }) + .filter((script) => { + return script.includes("git clone"); + }); +}; + +type Checkout = Parameters[0]; + +/** + * Run jobs that each call `checkout()` with their own options, side by side, + * in a pull request run or, with no repository in the trigger, a cron. + */ +const checkingOut = ( + ci: Ci, + jobs: Record, + event: typeof prEvent | typeof nightly = prEvent, +) => { + return running( + ci, + Object.fromEntries( + Object.entries(jobs).map(([id, opts]) => { + return [ + id, + async () => { + await checkout(opts); + }, + ]; + }), + ), + event, + ); +}; + +/** Run jobs side by side in a pull request run or a cron. */ +const running = ( + ci: Ci, + bodies: Record Promise>, + event: typeof prEvent | typeof nightly = prEvent, +) => { + const jobs = Object.entries(bodies).map(([id, body]) => { + return ci.job(id, body); + }); + + const options = + event === nightly + ? { + id: "nightly", + on: [{ event: "test/nightly" }], + check: false as const, + } + : { id: "pr", on: prTrigger }; + + return runFunction( + ci.pipeline(options, async () => { + await Promise.all( + jobs.map((job) => { + return job(); + }), + ); + }), + { event }, + ); +}; + +const refSteps = (stepIds: string[]) => { + return stepIds.filter((id) => { + return id.startsWith("github › ref:"); + }); +}; + +const requestsTo = (gh: FakeGitHub, path: string) => { + return gh.requests.filter((request) => { + return request.path === path; + }); +}; + +describe("checkout() of another repository", () => { + test("clones the commit the default branch resolves to", async () => { + const { api, gh, ci } = setup(); + + platform(gh); + + const result = await checkingOut(ci, { build: { repo: "acme/platform" } }); + + expect(result.type).toBe("function-resolved"); + expect(result.stepIds).toContain("github › default-branch:acme/platform"); + expect(result.stepIds).toContain("github › ref:acme/platform@main"); + expect(result.stepIds).toContain("build › checkout"); + + expect(cloneScripts(api)).toHaveLength(1); + expect(cloneScripts(api)[0]).toContain("checkout 'cafe1234'"); + expect(cloneScripts(api)[0]).not.toContain(secret); + + expect(result.steps["build › checkout"]).toMatchObject({ + sha: "cafe1234", + source: "github", + }); + }); + + test("keeps the installation token out of every step", async () => { + const { api, gh, ci } = setup(); + + platform(gh); + + const result = await checkingOut(ci, { + build: { repo: "acme/platform", ref: "v2" }, + }); + + expect(result.type).toBe("function-resolved"); + expect(JSON.stringify(result.steps)).not.toContain(secret); + expect(JSON.stringify(result.metadata)).not.toContain(secret); + expect(JSON.stringify(api.commands)).not.toContain(secret); + }); + + test("resolves a ref to a commit once per run", async () => { + const { api, gh, ci } = setup(); + + platform(gh); + + const result = await checkingOut(ci, { + first: { repo: "acme/platform", ref: "main" }, + second: { repo: "acme/platform", ref: "main", path: "/other" }, + }); + + expect(result.type).toBe("function-resolved"); + expect(refSteps(result.stepIds)).toEqual([ + "github › ref:acme/platform@main", + ]); + expect(requestsTo(gh, "/repos/acme/platform/commits/main")).toHaveLength(1); + expect(cloneScripts(api)).toHaveLength(2); + }); + + test("repo alone and repo with the default branch share one resolve step", async () => { + const { gh, ci } = setup(); + + platform(gh); + + const result = await checkingOut(ci, { + first: { repo: "acme/platform" }, + second: { repo: "acme/platform", ref: "main", path: "/other" }, + }); + + expect(result.type).toBe("function-resolved"); + expect(refSteps(result.stepIds)).toEqual([ + "github › ref:acme/platform@main", + ]); + expect(requestsTo(gh, "/repos/acme/platform/commits/main")).toHaveLength(1); + expect(result.steps["first › checkout"]).toMatchObject({ sha: "cafe1234" }); + expect(result.steps["second › checkout"]).toMatchObject({ + sha: "cafe1234", + }); + }); + + test("works when the run's trigger has no repository", async () => { + const { api, gh, ci } = setup(); + + platform(gh); + + const result = await checkingOut( + ci, + { build: { repo: "acme/platform", ref: "main" } }, + nightly, + ); + + expect(result.type).toBe("function-resolved"); + expect(cloneScripts(api)[0]).toContain("checkout 'cafe1234'"); + }); + + test("still needs a repository when none is given", async () => { + const { ci } = setup(); + + const result = await checkingOut(ci, { build: {} }, nightly); + + expect(result.type).toBe("function-rejected"); + expect(JSON.stringify(result.error)).toContain("needs a repository"); + }); + + test("a ref alone overrides the run's commit for its own repository", async () => { + const { api, gh, ci } = setup(); + + ownRepo(gh, "release", "feed9999"); + + const result = await checkingOut(ci, { build: { ref: "release" } }); + + expect(result.type).toBe("function-resolved"); + expect(cloneScripts(api)[0]).toContain("checkout 'feed9999'"); + }); + + test("a long ref keeps the step ID bounded", async () => { + const { gh, ci } = setup(); + const ref = "x".repeat(600); + + platform(gh); + + gh.route(`GET /repos/acme/platform/commits/${ref}`, { sha: "aaaa0000" }); + + const result = await checkingOut(ci, { + build: { repo: "acme/platform", ref }, + }); + + expect(result.type).toBe("function-resolved"); + + for (const id of result.stepIds) { + expect(id.length).toBeLessThanOrEqual(255); + } + }); + + test("fetches a fork's pull request head when the repo's case differs", async () => { + const { api, gh, ci } = setup(); + + gh.route("GET /repos/INNGEST/Inngest-JS/commits/abc1234", { + sha: "abc1234", + }); + + const result = await checkingOut( + ci, + { build: { repo: "INNGEST/Inngest-JS", ref: "abc1234" } }, + { + ...prEvent, + data: { + ...prEvent.data, + pull_request: { + ...prEvent.data.pull_request, + head: { + ...prEvent.data.pull_request.head, + repo: { full_name: "someone/inngest-js" }, + }, + }, + }, + }, + ); + + expect(result.type).toBe("function-resolved"); + expect(cloneScripts(api)[0]).toContain("fetch origin 'refs/pull/7/head'"); + }); +}); + +describe("what a failed lookup says", () => { + const cases: { + name: string; + routes?: (gh: FakeGitHub) => void; + opts?: Checkout; + provider?: Parameters[0]; + contains: string[]; + excludes?: string[]; + }[] = [ + { + name: "a ref that doesn't exist is named", + routes: (gh) => { + platform(gh); + + gh.route("GET /repos/acme/platform/commits/nope", {}, 422); + }, + opts: { repo: "acme/platform", ref: "nope" }, + contains: ["`nope`", "acme/platform"], + }, + { + name: "no default branch says so, not undefined", + routes: (gh) => { + gh.route("GET /repos/acme/platform", {}); + }, + contains: ["the default branch"], + excludes: ["undefined"], + }, + { + name: "an empty repository is called empty", + routes: (gh) => { + platform(gh); + + gh.route( + "GET /repos/acme/platform/commits/main", + { message: "Git Repository is empty." }, + 409, + ); + }, + contains: ["the repository is empty"], + }, + { + name: "the token provider says the token, not the GitHub App", + routes: (gh) => { + gh.route("GET /repos/acme/platform", { message: "Not Found" }, 404); + }, + contains: ["The token can't access"], + excludes: ["GitHub App"], + }, + ...[ + "acme", + "acme/platform/extra", + "-acme/platform", + "ac me/platform", + "acme/plat form", + "acme/..", + "acme/.", + "acme/pla\ttform", + "ac_me/platform", + ].map((repo) => { + return { + name: `the repo ${JSON.stringify(repo)} is rejected`, + opts: { repo }, + contains: ["`repo` must be"], + }; + }), + ]; + + test.each(cases)( + "$name", + async ({ + routes, + opts = { repo: "acme/platform" }, + provider, + contains, + excludes = [], + }) => { + const { ci, gh } = setup(provider); + + routes?.(gh); + + const result = await checkingOut(ci, { build: opts }); + + expect(result.type).toBe("function-rejected"); + + const message = JSON.stringify(result.error); + + for (const text of contains) { + expect(message).toContain(text); + } + + for (const text of excludes) { + expect(message).not.toContain(text); + } + }, + ); +}); + +describe("the GitHub App's installation", () => { + const key = generateKeyPairSync("rsa", { + modulusLength: 2048, + publicKeyEncoding: { type: "spki", format: "pem" }, + privateKeyEncoding: { type: "pkcs8", format: "pem" }, + }).privateKey; + + const app = (gh: FakeGitHub) => { + return githubApp({ + appId: "123", + privateKey: key, + baseUrl: "https://api.github.test", + fetch: gh.fetch, + }); + }; + + const tokenRoutes = (gh: FakeGitHub) => { + const expires = new Date(Date.now() + 3600_000).toISOString(); + + gh.route("POST /app/installations/1/access_tokens", { + token: "tok_run", + expires_at: expires, + }); + + gh.route("POST /app/installations/99/access_tokens", { + token: secret, + expires_at: expires, + }); + }; + + const requestsOf = (gh: FakeGitHub) => { + return gh.requests.map((request) => { + return `${request.method} ${request.path}`; + }); + }; + + /** Answer as the other installation would, and 404 for the run's own. */ + const onlyInstallation99 = (gh: FakeGitHub, path: string, body: unknown) => { + gh.handle(`GET ${path}`, (request) => { + return request.authorization?.includes("tok_run") + ? { body: { message: "Not Found" }, status: 404 } + : { body }; + }); + }; + + const rateLimited = (headers: Record) => { + return () => { + return { + body: { message: "API rate limit exceeded" }, + status: 403, + headers: { "x-ratelimit-remaining": "0", ...headers }, + }; + }; + }; + + const checkoutRepo = (ci: Ci, repo = "acme/platform") => { + return checkingOut(ci, { build: { repo } }); + }; + + test("the run's own installation is used first", async () => { + const { api, gh, ci } = setup(app); + + platform(gh); + tokenRoutes(gh); + + const result = await checkoutRepo(ci); + + expect(result.type).toBe("function-resolved"); + expect(cloneScripts(api)[0]).toContain("checkout 'cafe1234'"); + + const paths = requestsOf(gh); + + expect(paths).toContain("POST /app/installations/1/access_tokens"); + expect(paths).not.toContain("GET /repos/acme/platform/installation"); + expect(paths).not.toContain("POST /app/installations/99/access_tokens"); + }); + + test("tokens are narrowed to the one repository", async () => { + const { gh, ci } = setup(app); + + platform(gh); + tokenRoutes(gh); + + await checkoutRepo(ci); + + const minted = gh.requests.filter((request) => { + return ( + request.method === "POST" && request.path.endsWith("/access_tokens") + ); + }); + + expect(minted.length).toBeGreaterThan(0); + + for (const request of minted) { + expect(request.body).toMatchObject({ repositories: ["platform"] }); + } + }); + + test("is found from the repository when the run's can't see it", async () => { + const { api, gh, ci } = setup(app); + + tokenRoutes(gh); + + gh.route("GET /repos/acme/platform/installation", { id: 99 }); + + onlyInstallation99(gh, "/repos/acme/platform", { default_branch: "main" }); + + onlyInstallation99(gh, "/repos/acme/platform/commits/main", { + sha: "cafe1234", + }); + + const result = await checkoutRepo(ci); + + expect(result.type).toBe("function-resolved"); + expect(cloneScripts(api)[0]).toContain("checkout 'cafe1234'"); + + const paths = requestsOf(gh); + + expect(paths).toContain("GET /repos/acme/platform/installation"); + expect(paths).toContain("POST /app/installations/99/access_tokens"); + + expect(result.steps["github › ref:acme/platform@main"]).toMatchObject({ + installationId: 99, + sha: "cafe1234", + }); + + expect(JSON.stringify(result.steps)).not.toContain(secret); + }); + + test("an app that can't access the repository says so", async () => { + const { api, gh, ci } = setup(app); + + tokenRoutes(gh); + + gh.route("GET /repos/acme/secret", { message: "Not Found" }, 404); + + gh.route( + "GET /repos/acme/secret/installation", + { message: "Not Found" }, + 404, + ); + + const result = await checkoutRepo(ci, "acme/secret"); + + expect(result.type).toBe("function-rejected"); + expect(result.retriable).toBe(false); + + const message = JSON.stringify(result.error); + + expect(message).toContain("acme/secret"); + expect(message).toContain("GitHub App"); + + expect(cloneScripts(api)).toHaveLength(0); + }); + + test("a rate limit retries instead of reading as no access", async () => { + const { gh, ci } = setup(app); + + tokenRoutes(gh); + + gh.handle("GET /repos/acme/platform", rateLimited({ "retry-after": "1" })); + + const result = await checkoutRepo(ci); + + expect(result.type).toBe("function-rejected"); + expect(JSON.stringify(result.error)).toContain("rate limit"); + expect(JSON.stringify(result.error)).not.toContain("can't access"); + + expect( + requestsOf(gh).filter((path) => { + return path === "GET /repos/acme/platform"; + }).length, + ).toBeGreaterThan(1); + }); + + test("a rate limit looking up the installation retries too", async () => { + const { gh, ci } = setup(app); + + tokenRoutes(gh); + + gh.route("GET /repos/acme/platform", { message: "Not Found" }, 404); + + gh.handle("GET /repos/acme/platform/installation", rateLimited({})); + + const result = await checkoutRepo(ci); + + expect(result.type).toBe("function-rejected"); + expect(JSON.stringify(result.error)).toContain("rate limit"); + expect(JSON.stringify(result.error)).not.toContain("can't access"); + }); + + test("a rejected App says its credentials are the problem", async () => { + const { gh, ci } = setup(app); + + tokenRoutes(gh); + + gh.route("GET /repos/acme/platform", { message: "Not Found" }, 404); + + gh.route( + "GET /repos/acme/platform/installation", + { message: "Bad credentials" }, + 401, + ); + + const result = await checkoutRepo(ci); + + expect(result.type).toBe("function-rejected"); + expect(result.retriable).toBe(false); + expect(JSON.stringify(result.error)).toContain("credentials"); + }); +}); + +describe("files() of another repository or ref", () => { + /** A job cached on `files()`, in a run of its own. */ + const runImage = (ci: Ci, part: ReturnType) => { + const image = ci.job({ id: "image", cache: { key: part } }, async () => { + await $`build image`; + }); + + return runFunction( + ci.pipeline({ id: "pr", on: prTrigger }, async () => { + return image(); + }), + { event: prEvent }, + ); + }; + + /** Names of the cached snapshots after one run with the given tree. */ + const cachedNames = async (base: string, ref?: string) => { + const { api, gh, ci } = setup(); + + platform(gh); + + gh.route("GET /repos/acme/platform/git/trees/cafe1234", { + tree: [ + { type: "blob", path: "images/node-base/Dockerfile", sha: base }, + { type: "blob", path: "README.md", sha: "ignored" }, + ], + }); + + const result = await runImage( + ci, + files("images/node-base/**", { + repo: "acme/platform", + ...(ref ? { ref } : {}), + }), + ); + + expect(result.type).toBe("function-resolved"); + + return { + names: [...api.snapshots.values()].flatMap((snapshot) => { + return snapshot.name ? [snapshot.name] : []; + }), + paths: gh.requests.map((request) => { + return request.path; + }), + }; + }; + + test("reads the tree of the resolved commit, not the ref name", async () => { + const { paths } = await cachedNames("blob1", "main"); + + expect(paths).toContain("/repos/acme/platform/git/trees/cafe1234"); + expect(paths).not.toContain("/repos/acme/platform/git/trees/main"); + }); + + test("the key changes when the commit's matched files change", async () => { + const before = await cachedNames("blob1", "main"); + const same = await cachedNames("blob1", "main"); + const after = await cachedNames("blob2", "main"); + + expect(before.names).toHaveLength(1); + expect(same.names).toEqual(before.names); + expect(after.names).not.toEqual(before.names); + }); + + test("defaults the ref to the repository's default branch", async () => { + const { paths } = await cachedNames("blob1"); + + expect(paths).toContain("/repos/acme/platform/commits/main"); + }); + + test("a truncated tree throws instead of hashing part of the files", async () => { + const { gh, ci } = setup(); + + platform(gh); + + gh.route("GET /repos/acme/platform/git/trees/cafe1234", { + truncated: true, + tree: [{ type: "blob", path: "a", sha: "1" }], + }); + + const result = await runImage(ci, files("**", { repo: "acme/platform" })); + + expect(result.type).toBe("function-rejected"); + expect(result.retriable).toBe(false); + + const message = JSON.stringify(result.error); + + expect(message).toContain("too large to hash"); + expect(message).toContain("narrower patterns"); + }); + + test("keeps the part's own options", () => { + expect(files("a", "b", { repo: "acme/platform", ref: "main" })).toEqual({ + kind: "inngest/ci.cacheKeyPart", + type: "files", + patterns: ["a", "b"], + repo: "acme/platform", + ref: "main", + }); + + expect(files("a")).toEqual({ + kind: "inngest/ci.cacheKeyPart", + type: "files", + patterns: ["a"], + }); + }); +}); + +describe("a local run", () => { + let dir: string | undefined; + + afterEach(() => { + if (dir) { + rmSync(dir, { recursive: true, force: true }); + } + + dir = undefined; + }); + + const localEvent = () => { + dir = mkdtempSync(join(tmpdir(), "inngest-ci-source-")); + + execFileSync("git", ["init", "-q", "-b", "main"], { cwd: dir }); + + writeFileSync(join(dir, "a.txt"), "a"); + + return { + ...prEvent, + data: { ...prEvent.data, local: { path: dir, baseRef: "main" } }, + }; + }; + + const runLocal = (ci: Ci, opts: Checkout[]) => { + const build = ci.job("build", async () => { + for (const each of opts) { + await checkout(each); + } + }); + + return runFunction( + ci.pipeline({ id: "pr", on: prTrigger }, async () => { + return build(); + }), + { event: localEvent() }, + ); + }; + + test("uploads the working tree without options", async () => { + const { api, gh, ci } = setup(); + + const result = await runLocal(ci, [{}, { repo: "inngest/inngest-js" }]); + + expect(result.type).toBe("function-resolved"); + expect(result.steps["build › checkout"]).toMatchObject({ source: "local" }); + + expect(cloneScripts(api)).toHaveLength(0); + expect(gh.requests).toHaveLength(0); + }); + + test("clones from GitHub for another repository", async () => { + const { api, gh, ci } = setup(); + + platform(gh); + + const result = await runLocal(ci, [{ repo: "acme/platform" }]); + + expect(result.type).toBe("function-resolved"); + expect(cloneScripts(api)[0]).toContain("checkout 'cafe1234'"); + }); + + test("clones from GitHub for an explicit ref", async () => { + const { api, gh, ci } = setup(); + + ownRepo(gh, "release", "feed9999"); + + const result = await runLocal(ci, [{ ref: "release" }]); + + expect(result.type).toBe("function-resolved"); + expect(cloneScripts(api)[0]).toContain("checkout 'feed9999'"); + }); +}); diff --git a/packages/ci/src/github/source.ts b/packages/ci/src/github/source.ts new file mode 100644 index 000000000..a1609f1ec --- /dev/null +++ b/packages/ci/src/github/source.ts @@ -0,0 +1,326 @@ +/** + * Which repository and commit `checkout()` and `files()` read: the run's own, + * or another repository and ref resolved to a commit by memoized steps. + * + * @module + */ + +import { NonRetriableError } from "inngest"; +import { CiUsageError } from "../errors.ts"; +import { ciRun } from "../pipeline/metadata.ts"; +import { steps } from "../pipeline/names.ts"; +import type { CiRunScope } from "../pipeline/scope.ts"; +import { inGitHubSpan } from "../pipeline/scope.ts"; +import type { FilesOptions } from "../types.ts"; +import { once, parseRepo } from "../util.ts"; +import type { AuthContext, GitHubProvider, Octokit } from "./auth.ts"; +import { mapGitHubError } from "./rest.ts"; + +/** What a caller asked to read: a repository, a ref, both or neither. */ +export type SourceSpec = FilesOptions; + +/** A repository at one commit, and the installation that can read it. */ +export interface ResolvedSource { + owner: string; + name: string; + fullName: string; + /** The commit, never a ref name. */ + sha: string; + /** The GitHub App installation that can read it. Not a secret. */ + installationId?: number; +} + +/** Whether two `owner/name` strings name the same repository, ignoring case. */ +export const sameRepo = (a: string, b: string | undefined): boolean => { + return a.toLowerCase() === b?.toLowerCase(); +}; + +/** + * Whether a spec means the run's own repository at the run's own commit, which + * needs no lookup and, locally, is the working tree. + */ +export const targetsRunRepo = (run: CiRunScope, spec: SourceSpec): boolean => { + return ( + spec.ref === undefined && + (spec.repo === undefined || sameRepo(spec.repo, run.repo?.fullName)) + ); +}; + +const sourceKey = (fullName: string, ref: string): string => { + return `${fullName.toLowerCase()}@${ref}`; +}; + +const accessError = (fullName: string, viaApp: boolean): NonRetriableError => { + return new NonRetriableError( + viaApp + ? `The GitHub App can't access \`${fullName}\`. Give the GitHub App access to that repository, by installing it on the owner or adding the repository to its installation.` + : `The token can't access \`${fullName}\`. Use a token that has access to that repository.`, + ); +}; + +const notFoundError = ( + fullName: string, + ref: string | undefined, +): NonRetriableError => { + return new NonRetriableError( + ref === undefined + ? `Couldn't find the default branch of \`${fullName}\`.` + : `\`${ref}\` isn't a branch, tag or commit of \`${fullName}\`.`, + ); +}; + +const emptyError = (fullName: string): NonRetriableError => { + return new NonRetriableError( + `\`${fullName}\` has no commits to read: the repository is empty.`, + ); +}; + +const providerOf = (run: CiRunScope): GitHubProvider => { + const provider = run.ci.github as GitHubProvider | undefined; + + if (!provider) { + throw new CiUsageError( + "No GitHub provider is configured. Pass `github: githubApp({ … })` or `githubToken({ … })` to `createCi`.", + ); + } + + return provider; +}; + +/** + * How to authenticate to read a source. Another repository than the run's gets + * a token narrowed to just that repository. + */ +const authFor = (run: CiRunScope, source: ResolvedSource): AuthContext => { + return { + owner: source.owner, + repo: source.name, + ...(source.installationId === undefined + ? {} + : { installationId: source.installationId }), + ...(sameRepo(source.fullName, run.repo?.fullName) + ? {} + : { repositoryNames: [source.name] }), + }; +}; + +/** The Octokit client for a resolved source, which may be another installation. */ +export const octokitForSource = ( + run: CiRunScope, + source: ResolvedSource, +): Promise => { + return providerOf(run).octokit(authFor(run, source)); +}; + +/** The token that clones a resolved source. Call it only inside a step handler. */ +export const tokenForSource = ( + run: CiRunScope, + source: ResolvedSource, +): Promise => { + return providerOf(run).token(authFor(run, source)); +}; + +const statusOf = (error: unknown): number | undefined => { + return (error as { status?: number }).status; +}; + +/** + * Return when an error means GitHub can't see the repository (a 403 that isn't + * a rate limit, or a 404). Throw anything else: rate limits as retries, server + * errors as they are, and other client errors as non-retriable. + */ +const throwUnlessMissing = (error: unknown): void => { + const mapped = mapGitHubError(error); + const status = statusOf(error); + + if ( + mapped instanceof NonRetriableError && + (status === 403 || status === 404) + ) { + return; + } + + throw mapped ?? error; +}; + +/** + * Run a read against a repository with an installation that can see it. The + * run's own installation goes first, since that's usually enough, and the + * repository's own installation is only looked up when that can't see it. + */ +const locate = async ( + run: CiRunScope, + fullName: string, + read: (octokit: Octokit, source: ResolvedSource) => Promise, +): Promise<{ value: T; source: ResolvedSource }> => { + const { owner, name } = parseRepo(fullName); + const { installationFor } = providerOf(run); + const first = installationFor ? run.repo?.installationId : undefined; + + const attempt = async (installationId: number | undefined) => { + const source: ResolvedSource = { + owner, + name, + fullName, + sha: "", + ...(installationId === undefined ? {} : { installationId }), + }; + + const octokit = await octokitForSource(run, source); + + return { value: await read(octokit, source), source }; + }; + + if (!installationFor || first !== undefined) { + try { + return await attempt(first); + } catch (error) { + throwUnlessMissing(error); + } + } + + if (!installationFor) { + throw accessError(fullName, false); + } + + const found = await installationFor(owner, name); + + if (found === undefined || found === first) { + throw accessError(fullName, true); + } + + try { + return await attempt(found); + } catch (error) { + throwUnlessMissing(error); + + throw accessError(fullName, true); + } +}; + +/** The commit a ref points at, in a repository this client can read. */ +const readCommit = async ( + octokit: Octokit, + source: ResolvedSource, + ref: string, +): Promise => { + try { + const { data } = await octokit.rest.repos.getCommit({ + owner: source.owner, + repo: source.name, + ref, + }); + + return data.sha; + } catch (error) { + const status = statusOf(error); + + if (status === 409) { + throw emptyError(source.fullName); + } + + if (status === 422) { + throw notFoundError(source.fullName, ref); + } + + if (status === 404) { + // A missing ref and a repository this client can't see are both a 404. + // Asking for the repository says which. + await octokit.rest.repos.get({ owner: source.owner, repo: source.name }); + + throw notFoundError(source.fullName, ref); + } + + throw error; + } +}; + +/** + * The default branch's name, from one step per repository per run. `repo` and + * `ref: ""` then share one commit step, keyed on the branch. + */ +const defaultBranch = (run: CiRunScope, fullName: string): Promise => { + return once(run.defaultBranches, fullName.toLowerCase(), () => { + return inGitHubSpan(run, () => { + return ciRun(run, steps.resolveDefaultBranch(fullName), async () => { + const { value } = await locate( + run, + fullName, + async (octokit, source) => { + const { data } = await octokit.rest.repos.get({ + owner: source.owner, + repo: source.name, + }); + + return data.default_branch; + }, + ); + + if (!value) { + throw notFoundError(fullName, undefined); + } + + return value; + }); + }); + }); +}; + +/** + * Resolve what a caller asked to read to a commit. + * + * The run's own repository and commit come straight from the run. Anything + * else is looked up in memoized steps, in the run's GitHub span rather than + * whichever job asked first, so every replay and every job sees the same + * commit: one step finds a repository's default branch when no `ref` is given, + * and one step per repository and branch finds the commit. Call it outside a + * step: a step can't start another. Called again for what it already resolved, + * it starts nothing, so a step can call it. + */ +export const resolveSource = async ( + run: CiRunScope, + spec: SourceSpec, +): Promise => { + if (targetsRunRepo(run, spec)) { + if (!run.repo) { + throw new CiUsageError( + 'This run\'s trigger has no repository. Pass `repo: "owner/name"`, or set `repo` on the pipeline.', + ); + } + + return run.repo; + } + + const fullName = spec.repo ?? run.repo?.fullName; + + if (!fullName) { + throw new CiUsageError( + 'This run\'s trigger has no repository, so `ref` needs a `repo: "owner/name"` beside it.', + ); + } + + parseRepo(fullName); + + // Checked here, outside the steps, so a missing provider is a usage error + // and not something a step retries. + providerOf(run); + + const ref = spec.ref ?? (await defaultBranch(run, fullName)); + const key = sourceKey(fullName, ref); + + return once(run.sources, key, () => { + return inGitHubSpan(run, () => { + return ciRun(run, steps.resolveRef(key), async () => { + const { value, source } = await locate( + run, + fullName, + (octokit, found) => { + return readCommit(octokit, found, ref); + }, + ); + + return { ...source, sha: value }; + }); + }); + }); +}; diff --git a/packages/ci/src/index.ts b/packages/ci/src/index.ts index 05369eb6d..8cf64bbea 100644 --- a/packages/ci/src/index.ts +++ b/packages/ci/src/index.ts @@ -101,6 +101,7 @@ export type { Duration, EventDataOf, ExtraMachine, + FilesOptions, FlowControlOptions, From, Job, diff --git a/packages/ci/src/pipeline/names.ts b/packages/ci/src/pipeline/names.ts index a58ab1868..d682a9d0c 100644 --- a/packages/ci/src/pipeline/names.ts +++ b/packages/ci/src/pipeline/names.ts @@ -11,7 +11,7 @@ */ import type { CheckConclusion } from "../types.ts"; -import { formatDuration } from "../util.ts"; +import { boundedName, formatDuration } from "../util.ts"; import { version } from "../version.ts"; import type { StepTag } from "./metadata.ts"; import type { CiRunScope } from "./scope.ts"; @@ -262,6 +262,22 @@ export const steps = { }; }, + resolveDefaultBranch: (fullName: string): StepSpec => { + return { + id: boundedName( + joinId("github", `default-branch:${fullName.toLowerCase()}`), + ), + name: "Resolve ref", + }; + }, + + resolveRef: (key: string): StepSpec => { + return { + id: boundedName(joinId("github", `ref:${key}`)), + name: "Resolve ref", + }; + }, + commentNotAllowed: (): StepSpec => { return { id: joinId("github", "comment:denied"), diff --git a/packages/ci/src/pipeline/pipeline.ts b/packages/ci/src/pipeline/pipeline.ts index 8c8eb2c9e..c42184a4e 100644 --- a/packages/ci/src/pipeline/pipeline.ts +++ b/packages/ci/src/pipeline/pipeline.ts @@ -44,7 +44,7 @@ import type { PipelineContext, RepoContext, } from "../types.ts"; -import { formatDuration } from "../util.ts"; +import { formatDuration, parseRepo } from "../util.ts"; import type { CacheBuildData } from "./cacheBuild.ts"; import type { RegisteredJob } from "./job.ts"; import { conclusionForError, runJob } from "./job.ts"; @@ -222,6 +222,8 @@ const newRunScope = ({ ...(repo ? { repo } : {}), ...(build ? { build } : {}), builds: new Map(), + sources: new Map(), + defaultBranches: new Map(), jobCalls: new Map(), createdSnapshots: new Set(), summaries: [], @@ -250,19 +252,6 @@ const newRunScope = ({ }; }; -/** Split a configured `repo` into its owner and name. */ -const parseRepo = (fullName: string): { owner: string; name: string } => { - const [owner, name, ...rest] = fullName.split("/"); - - if (!owner || !name || rest.length > 0) { - throw new CiUsageError( - `\`repo\` must be "owner/name", but got "${fullName}".`, - ); - } - - return { owner, name }; -}; - /** * Run a pipeline: set up the run scope, report the pipeline check, run the * handler, then always complete the check and destroy the machines. diff --git a/packages/ci/src/pipeline/scope.ts b/packages/ci/src/pipeline/scope.ts index b4f7932b4..13e0d60d3 100644 --- a/packages/ci/src/pipeline/scope.ts +++ b/packages/ci/src/pipeline/scope.ts @@ -10,6 +10,7 @@ import type { AsyncContext, DurableSandboxTools } from "inngest/experimental"; import { getAsyncCtx, runWithAsyncCtx } from "inngest/experimental"; import type { CachedSnapshot } from "../cache/cache.ts"; import { CiUsageError } from "../errors.ts"; +import type { ResolvedSource } from "../github/source.ts"; import type { CheckAnnotation, CheckConclusion, @@ -40,6 +41,20 @@ export const rebuildSuffix = " (rebuild)"; /** The default working directory, which is where `checkout()` puts the repo. */ export const defaultCwd = "/work"; +/** + * The local working tree this run reads, when it has one. A run with local + * repository data still talks to GitHub when `INNGEST_CI_GITHUB=live`. + */ +export const localRepo = ( + run: Pick, +): NonNullable | undefined => { + if (process.env.INNGEST_CI_GITHUB === "live") { + return undefined; + } + + return run.repo?.local; +}; + /** * Where a matrix combination's job config remembers the matrix and combination * it came from, so its snapshot is built by the matrix's build function. @@ -180,6 +195,18 @@ export interface CiRunScope { * else, and no job reads another job's state through it. */ builds: Map>; + /** + * The repositories and refs `checkout()` and `files()` have resolved to a + * commit, keyed by repository and ref, so each is resolved by one step per + * run. It holds promises and nothing else. + */ + sources: Map>; + /** + * The default branch of each repository `checkout()` and `files()` have + * asked for without a `ref`, keyed by repository, so a repository's default + * branch is found by one step per run. It holds promises and nothing else. + */ + defaultBranches: Map>; /** How many direct calls of each job have started, keyed by job ID. */ jobCalls: Map; /** diff --git a/packages/ci/src/testing/fakeGitHub.ts b/packages/ci/src/testing/fakeGitHub.ts index c6633884c..a53e6a530 100644 --- a/packages/ci/src/testing/fakeGitHub.ts +++ b/packages/ci/src/testing/fakeGitHub.ts @@ -8,6 +8,14 @@ export interface FakeGitHubRequest { method: string; path: string; body?: unknown; + /** The request's `Authorization` header, if it had one. */ + authorization?: string; +} + +export interface FakeGitHubReply { + body: unknown; + status?: number; + headers?: Record; } export interface FakeGitHub { @@ -15,6 +23,11 @@ export interface FakeGitHub { requests: FakeGitHubRequest[]; /** Reply to `METHOD /path` with this body. Paths may end in `*`. */ route(pattern: string, body: unknown, status?: number): void; + /** Like `route`, but decides from the request, such as who is asking. */ + handle( + pattern: string, + reply: (request: FakeGitHubRequest) => FakeGitHubReply, + ): void; } /** @@ -22,7 +35,10 @@ export interface FakeGitHub { */ export const createFakeGitHub = (): FakeGitHub => { const requests: FakeGitHubRequest[] = []; - const routes: Array<{ pattern: string; body: unknown; status: number }> = []; + const routes: Array<{ + pattern: string; + reply: (request: FakeGitHubRequest) => FakeGitHubReply; + }> = []; const fakeFetch = (async ( input: string | URL | Request, @@ -32,11 +48,16 @@ export const createFakeGitHub = (): FakeGitHub => { const method = (init?.method ?? "GET").toUpperCase(); const key = `${method} ${url.pathname}`; - requests.push({ + const authorization = new Headers(init?.headers).get("authorization"); + + const request: FakeGitHubRequest = { method, path: url.pathname, ...(init?.body ? { body: JSON.parse(String(init.body)) } : {}), - }); + ...(authorization ? { authorization } : {}), + }; + + requests.push(request); const route = routes.find(({ pattern }) => { return pattern.endsWith("*") @@ -44,9 +65,11 @@ export const createFakeGitHub = (): FakeGitHub => { : pattern === key; }); - const response = new Response(JSON.stringify(route?.body ?? {}), { - status: route?.status ?? 200, - headers: { "Content-Type": "application/json" }, + const reply = route?.reply(request); + + const response = new Response(JSON.stringify(reply?.body ?? {}), { + status: reply?.status ?? 200, + headers: { "Content-Type": "application/json", ...reply?.headers }, }); // Octokit's pagination reads the request URL from the response. @@ -59,7 +82,15 @@ export const createFakeGitHub = (): FakeGitHub => { fetch: fakeFetch, requests, route: (pattern, body, status = 200) => { - routes.unshift({ pattern, body, status }); + routes.unshift({ + pattern, + reply: () => { + return { body, status }; + }, + }); + }, + handle: (pattern, reply) => { + routes.unshift({ pattern, reply }); }, }; }; diff --git a/packages/ci/src/types.ts b/packages/ci/src/types.ts index a09e84dd0..a198e3d65 100644 --- a/packages/ci/src/types.ts +++ b/packages/ci/src/types.ts @@ -149,12 +149,23 @@ export interface RepoContext { /** * A part of a cache key that's resolved at runtime, like `files()`. */ -export interface CacheKeyPart { +export interface CacheKeyPart extends Readonly { readonly kind: "inngest/ci.cacheKeyPart"; readonly type: "files"; readonly patterns: string[]; } +/** Where `files()` reads from, when that isn't the run's own commit. */ +export interface FilesOptions { + /** The repository to read the files from, as `owner/name`. Defaults to the run's. */ + repo?: string; + /** + * The branch, tag or commit to read them at. Defaults to the run's commit, or + * to the repository's default branch when `repo` is another repository. + */ + ref?: string; +} + /** * What a job's cache is keyed on: a string, a `files()` part, a list of those * joined together, or a function that computes the key when the job runs. diff --git a/packages/ci/src/util.ts b/packages/ci/src/util.ts index 1dcbbfd23..9de2226a5 100644 --- a/packages/ci/src/util.ts +++ b/packages/ci/src/util.ts @@ -385,3 +385,52 @@ export const git = async (cwd: string, args: string[]): Promise => { export const errorMessage = (error: unknown): string => { return error instanceof Error ? error.message : String(error); }; + +const ownerPattern = /^[A-Za-z0-9](?:[A-Za-z0-9-]*)$/; + +const namePattern = /^[A-Za-z0-9._-]+$/; + +/** Split `owner/name`, or say what a repository should look like. */ +export const parseRepo = ( + fullName: string, +): { owner: string; name: string } => { + const [owner, name, ...rest] = fullName.split("/"); + + if ( + !owner || + !name || + rest.length > 0 || + !ownerPattern.test(owner) || + !namePattern.test(name) || + name === "." || + name === ".." + ) { + throw new CiUsageError( + `\`repo\` must be "owner/name", but got "${fullName}".`, + ); + } + + return { owner, name }; +}; + +/** + * The value kept under `key`, made by `make` the first time it's asked for. + * Kept values are promises, so every caller of a key awaits the same work. + */ +export const once = ( + kept: Map, + key: string, + make: () => T, +): T => { + const known = kept.get(key); + + if (known !== undefined) { + return known; + } + + const made = make(); + + kept.set(key, made); + + return made; +}; From d42ab5beb30877a9f24a9ccf0ca3dfdc8764fd6b Mon Sep 17 00:00:00 2001 From: Jack Williams Date: Tue, 13 Oct 2026 21:01:13 +0100 Subject: [PATCH 07/18] Update ci-trace-spans.md --- .changeset/ci-trace-spans.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/ci-trace-spans.md b/.changeset/ci-trace-spans.md index ac0d50c6a..3449b6911 100644 --- a/.changeset/ci-trace-spans.md +++ b/.changeset/ci-trace-spans.md @@ -2,6 +2,6 @@ "@inngest/ci": minor --- -Group a run's steps into trace spans (one per job, one per command, one per extra machine), name CI's own steps for what they do, and mark them as CI's work so a trace can tell them from yours. Give a job a `name` to change what its span is called. Every span has a kind (job, sandbox, command, snapshot, github or attempt), and each statement on a background process (`output`, `exited`, `kill`) is a row of its own. Saving a sandbox is one `snapshot` row. +Group a run's steps into trace spans (one per job, one per command, one per extra machine), name CI's own steps for what they do, and mark them as CI's work so a trace can tell them from yours. Saving a sandbox is one `snapshot` row. `@inngest/ci` now needs `inngest` 4.23.1 or later, which takes a snapshot's own trace span and always has the span API; older SDKs are no longer supported. From 477822cfe0f306aa99a52aae2a6bdf984d420aa1 Mon Sep 17 00:00:00 2001 From: x Date: Tue, 13 Oct 2026 20:08:08 +0000 Subject: [PATCH 08/18] Name a job's span from its config, keeping the number of later direct calls A later direct call has the path `id (n)`, which is not a registry key, so its span lost the configured name. The span now takes the name from the job's config and keeps the call number. Co-Authored-By: Claude Opus 5.5 --- packages/ci/src/machine/machine.ts | 2 +- packages/ci/src/pipeline/job.ts | 2 +- packages/ci/src/pipeline/names.ts | 20 +++++++++++++------- packages/ci/src/pipeline/scope.ts | 3 ++- packages/ci/src/pipeline/spans.test.ts | 16 +++++++++++++++- 5 files changed, 32 insertions(+), 11 deletions(-) diff --git a/packages/ci/src/machine/machine.ts b/packages/ci/src/machine/machine.ts index 9a36a46f1..1728667c4 100644 --- a/packages/ci/src/machine/machine.ts +++ b/packages/ci/src/machine/machine.ts @@ -43,7 +43,7 @@ import { export const ensureMachine = async ( scope: CiJobScope, ): Promise => { - scope.machine ??= inJobSpan(scope.run, scope.jobPath, () => { + scope.machine ??= inJobSpan(scope.run, scope.config, scope.jobPath, () => { return inMachineSpan(scope, () => { const stepId = joinId(scope.path, "machine"); diff --git a/packages/ci/src/pipeline/job.ts b/packages/ci/src/pipeline/job.ts index e482b6d8d..77d3a6f35 100644 --- a/packages/ci/src/pipeline/job.ts +++ b/packages/ci/src/pipeline/job.ts @@ -398,7 +398,7 @@ export const validateInput = async ( const jobBody = ( args: RunJobArgs & { run: CiRunScope; path: string; number: number }, ): Promise => { - return inJobSpan(args.run, args.path, () => { + return inJobSpan(args.run, args.config, args.path, () => { return jobSteps(args); }); }; diff --git a/packages/ci/src/pipeline/names.ts b/packages/ci/src/pipeline/names.ts index 475a259b5..f2ab09725 100644 --- a/packages/ci/src/pipeline/names.ts +++ b/packages/ci/src/pipeline/names.ts @@ -10,10 +10,9 @@ * @module */ -import type { CheckConclusion } from "../types.ts"; +import type { CheckConclusion, JobConfig } from "../types.ts"; import { formatDuration } from "../util.ts"; import { version } from "../version.ts"; -import type { CiRunScope } from "./scope.ts"; import { joinId, scopeSeparator } from "./scope.ts"; import type { SpanInfo, SpanKind } from "./spans.ts"; import { originOption } from "./spans.ts"; @@ -43,8 +42,8 @@ const ciSpan = (kind: SpanKind, id: string, name: string): SpanInfo => { */ export const spans = { /** A job, which holds everything it does. */ - job: (run: CiRunScope, path: string): SpanInfo => { - return userSpan("job", path, traceName.job(run, path)); + job: (config: JobConfig, path: string): SpanInfo => { + return userSpan("job", path, traceName.job(config, path)); }, /** The run's one span for reporting to GitHub. */ @@ -110,9 +109,16 @@ const outcomes: Partial> = { }; export const traceName = { - /** A job's span: its `name`, or its path when it has none. */ - job: (run: CiRunScope, path: string): string => { - return run.ci.jobs.get(path)?.config.name ?? path; + /** + * A job's span: its `name`, or its path when it has none. A later direct + * call, whose path is `${id} (n)`, keeps its `(n)` after the name. + */ + job: (config: JobConfig, path: string): string => { + if (!config.name) { + return path; + } + + return `${config.name}${path.slice(config.id.length)}`; }, /** The span a job's sandbox starts in. */ diff --git a/packages/ci/src/pipeline/scope.ts b/packages/ci/src/pipeline/scope.ts index b4f7932b4..d960c8340 100644 --- a/packages/ci/src/pipeline/scope.ts +++ b/packages/ci/src/pipeline/scope.ts @@ -554,11 +554,12 @@ const prefixStepId = (idOrOptions: unknown, prefix: string): unknown => { */ export const inJobSpan = ( run: CiRunScope, + config: JobConfig, jobPath: string, fn: () => R, ): R => { return runWithAsyncCtx(run.asyncCtx, () => { - return inSpan(spans.job(run, jobPath), fn); + return inSpan(spans.job(config, jobPath), fn); }); }; diff --git a/packages/ci/src/pipeline/spans.test.ts b/packages/ci/src/pipeline/spans.test.ts index 6e5f13b71..5dd3b06f6 100644 --- a/packages/ci/src/pipeline/spans.test.ts +++ b/packages/ci/src/pipeline/spans.test.ts @@ -8,7 +8,8 @@ import { describe, expect, test, vi } from "vitest"; import { snapshotMachine } from "../machine/machine.ts"; -import { ciOrigin } from "./names.ts"; +import type { JobConfig } from "../types.ts"; +import { ciOrigin, traceName } from "./names.ts"; import type { CiJobScope } from "./scope.ts"; import { originOption } from "./spans.ts"; @@ -55,3 +56,16 @@ describe("a snapshot's span", () => { ]); }); }); + +describe("a job's span name", () => { + const named = { id: "build", name: "Build it" } as JobConfig; + + test("is the job's name, and its id when it has none", () => { + expect(traceName.job(named, "build")).toBe("Build it"); + expect(traceName.job({ id: "build" } as JobConfig, "build")).toBe("build"); + }); + + test("keeps the name on a later direct call, with its number", () => { + expect(traceName.job(named, "build (2)")).toBe("Build it (2)"); + }); +}); From 5d039f37e4e26b27a4f6d80e021dd2ea6fba69e8 Mon Sep 17 00:00:00 2001 From: Jack Williams Date: Tue, 13 Oct 2026 20:29:04 +0000 Subject: [PATCH 09/18] Drop changeset: ci-shared-helpers is a pure internal refactor Co-Authored-By: Claude Opus 5.5 --- .changeset/ci-shared-helpers.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .changeset/ci-shared-helpers.md diff --git a/.changeset/ci-shared-helpers.md b/.changeset/ci-shared-helpers.md deleted file mode 100644 index 400014067..000000000 --- a/.changeset/ci-shared-helpers.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@inngest/ci": patch ---- - -Internal refactor: CI's step IDs and names come from one catalog, and snapshot naming, sandbox paging and snapshot deletes, cache lookups and check updates share helpers. Nothing user-visible changes. From 77ecbcd4d51f35560f00b35911cf08cdfd34292b Mon Sep 17 00:00:00 2001 From: Jack Williams Date: Tue, 13 Oct 2026 22:20:48 +0000 Subject: [PATCH 10/18] docs(ci): rewrite checkout-repo changeset as a release note Co-Authored-By: Claude Sonnet 5.5 --- .changeset/ci-checkout-repo.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.changeset/ci-checkout-repo.md b/.changeset/ci-checkout-repo.md index c4dba711b..079233f65 100644 --- a/.changeset/ci-checkout-repo.md +++ b/.changeset/ci-checkout-repo.md @@ -2,4 +2,8 @@ "@inngest/ci": minor --- -`checkout()` and `files()` take `repo` and `ref`, to read another repository or ref. The ref is resolved to a commit once per run, and the GitHub App installation is found from the repository. `checkout({ repo })` also works in a run whose trigger has no repository. +`checkout()` and `files()` can now read another repository or ref, including in runs whose trigger has no repository. + +```ts +await checkout({ repo: "acme/shared-config", ref: "main" }); +``` From 24cecbde2a7e79a2f1ae042cd72c607b24fa4c98 Mon Sep 17 00:00:00 2001 From: Jack Williams Date: Tue, 13 Oct 2026 23:46:50 +0000 Subject: [PATCH 11/18] fix(ci): treat 422 as "not in this installation" The first lookup mints a token narrowed to the repository under the run's own installation. For a repository outside that installation GitHub answers 422, which wasn't swallowed, so the installation lookup never ran. Co-Authored-By: Claude Opus 5.5 --- packages/ci/src/github/source.test.ts | 24 ++++++++++++++++++++++++ packages/ci/src/github/source.ts | 5 +++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/packages/ci/src/github/source.test.ts b/packages/ci/src/github/source.test.ts index dc14ddb7e..fe8604245 100644 --- a/packages/ci/src/github/source.test.ts +++ b/packages/ci/src/github/source.test.ts @@ -539,6 +539,30 @@ describe("the GitHub App's installation", () => { expect(JSON.stringify(result.steps)).not.toContain(secret); }); + test("is found from the repository when the narrowed token is refused", async () => { + const { api, gh, ci } = setup(app); + + platform(gh); + tokenRoutes(gh); + + gh.route( + "POST /app/installations/1/access_tokens", + { message: "Validation Failed" }, + 422, + ); + + gh.route("GET /repos/acme/platform/installation", { id: 99 }); + + const result = await checkoutRepo(ci); + + expect(result.type).toBe("function-resolved"); + expect(cloneScripts(api)[0]).toContain("checkout 'cafe1234'"); + + expect(requestsOf(gh)).toContain( + "POST /app/installations/99/access_tokens", + ); + }); + test("an app that can't access the repository says so", async () => { const { api, gh, ci } = setup(app); diff --git a/packages/ci/src/github/source.ts b/packages/ci/src/github/source.ts index a1609f1ec..9371ed733 100644 --- a/packages/ci/src/github/source.ts +++ b/packages/ci/src/github/source.ts @@ -126,7 +126,8 @@ const statusOf = (error: unknown): number | undefined => { /** * Return when an error means GitHub can't see the repository (a 403 that isn't - * a rate limit, or a 404). Throw anything else: rate limits as retries, server + * a rate limit, a 404, or the 422 of a token request narrowed to a repository + * outside the installation). Throw anything else: rate limits as retries, server * errors as they are, and other client errors as non-retriable. */ const throwUnlessMissing = (error: unknown): void => { @@ -135,7 +136,7 @@ const throwUnlessMissing = (error: unknown): void => { if ( mapped instanceof NonRetriableError && - (status === 403 || status === 404) + (status === 403 || status === 404 || status === 422) ) { return; } From 317b1061492c6452632bd12bc342b8b355919a41 Mon Sep 17 00:00:00 2001 From: Jack Williams Date: Tue, 13 Oct 2026 23:47:44 +0000 Subject: [PATCH 12/18] fix(ci): validate files() repo when it is defined A malformed `repo` in `files()` only failed when the run resolved it. It now throws CiUsageError where the pipeline is defined, so it fails at boot. Co-Authored-By: Claude Opus 5.5 --- packages/ci/src/cache/cache.ts | 6 ++++++ packages/ci/src/github/source.test.ts | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/packages/ci/src/cache/cache.ts b/packages/ci/src/cache/cache.ts index bea1e23c7..b8a3487eb 100644 --- a/packages/ci/src/cache/cache.ts +++ b/packages/ci/src/cache/cache.ts @@ -36,6 +36,7 @@ import { formatRelative, hash, isSnapshotNotFound, + parseRepo, stableStringify, } from "../util.ts"; import { formatName } from "./names.ts"; @@ -814,6 +815,11 @@ export const files = ( const last = list.at(-1); const options = typeof last === "object" ? last : undefined; + // Checked here, where the pipeline is defined, so a bad `repo` fails at boot. + if (options?.repo) { + parseRepo(options.repo); + } + const patterns = list.filter((arg): arg is string => { return typeof arg === "string"; }); diff --git a/packages/ci/src/github/source.test.ts b/packages/ci/src/github/source.test.ts index fe8604245..44b163bec 100644 --- a/packages/ci/src/github/source.test.ts +++ b/packages/ci/src/github/source.test.ts @@ -15,6 +15,7 @@ import { join } from "node:path"; import { afterEach, describe, expect, test } from "vitest"; import { files } from "../cache/cache.ts"; import { checkout } from "../checkout/checkout.ts"; +import { CiUsageError } from "../errors.ts"; import { $ } from "../machine/command.ts"; import { createCi } from "../pipeline/createCi.ts"; import { createCiTestClient } from "../testing/client.ts"; @@ -738,6 +739,15 @@ describe("files() of another repository or ref", () => { expect(message).toContain("narrower patterns"); }); + test.each(["acme", "acme/platform/extra", "acme/..", "ac me/platform"])( + "rejects the repo %j when the part is defined", + (repo) => { + expect(() => { + return files("a", { repo }); + }).toThrow(CiUsageError); + }, + ); + test("keeps the part's own options", () => { expect(files("a", "b", { repo: "acme/platform", ref: "main" })).toEqual({ kind: "inngest/ci.cacheKeyPart", From 6fd3853ed48f089186c921b260a4c1a773ffc069 Mon Sep 17 00:00:00 2001 From: Jack Williams Date: Tue, 13 Oct 2026 23:48:13 +0000 Subject: [PATCH 13/18] fix(ci): allow underscores in repository owners Enterprise Managed User accounts (`name_shortcode`) own repositories, and `repo` accepted them before the owner pattern was added. Co-Authored-By: Claude Opus 5.5 --- packages/ci/src/github/source.test.ts | 7 ++++++- packages/ci/src/util.ts | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/ci/src/github/source.test.ts b/packages/ci/src/github/source.test.ts index 44b163bec..26a56388c 100644 --- a/packages/ci/src/github/source.test.ts +++ b/packages/ci/src/github/source.test.ts @@ -371,7 +371,6 @@ describe("what a failed lookup says", () => { "acme/..", "acme/.", "acme/pla\ttform", - "ac_me/platform", ].map((repo) => { return { name: `the repo ${JSON.stringify(repo)} is rejected`, @@ -748,6 +747,12 @@ describe("files() of another repository or ref", () => { }, ); + test("accepts an Enterprise Managed User owner", () => { + expect(() => { + return files("a", { repo: "octo_cat/platform" }); + }).not.toThrow(); + }); + test("keeps the part's own options", () => { expect(files("a", "b", { repo: "acme/platform", ref: "main" })).toEqual({ kind: "inngest/ci.cacheKeyPart", diff --git a/packages/ci/src/util.ts b/packages/ci/src/util.ts index 9de2226a5..d5104089c 100644 --- a/packages/ci/src/util.ts +++ b/packages/ci/src/util.ts @@ -386,7 +386,7 @@ export const errorMessage = (error: unknown): string => { return error instanceof Error ? error.message : String(error); }; -const ownerPattern = /^[A-Za-z0-9](?:[A-Za-z0-9-]*)$/; +const ownerPattern = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; const namePattern = /^[A-Za-z0-9._-]+$/; From 8aad8aaa101273390041360cbacb27f32be9412d Mon Sep 17 00:00:00 2001 From: Jack Williams Date: Tue, 13 Oct 2026 23:49:03 +0000 Subject: [PATCH 14/18] refactor(ci): replace checkout sources with a plain if checkout() has two ways to fill the machine, the local working tree and a GitHub clone. The CheckoutSource/CheckoutPlan abstraction existed for a speculative third. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- packages/ci/src/checkout/checkout.ts | 78 ++++++---------------------- 1 file changed, 15 insertions(+), 63 deletions(-) diff --git a/packages/ci/src/checkout/checkout.ts b/packages/ci/src/checkout/checkout.ts index 718d8b9b3..a4dee5679 100644 --- a/packages/ci/src/checkout/checkout.ts +++ b/packages/ci/src/checkout/checkout.ts @@ -15,7 +15,6 @@ import { } from "../github/source.ts"; import { ensureMachine } from "../machine/machine.ts"; import { ciRun } from "../pipeline/metadata.ts"; -import type { StepSpec } from "../pipeline/names.ts"; import { steps } from "../pipeline/names.ts"; import type { CiRunScope, MachineHandle } from "../pipeline/scope.ts"; import { @@ -117,17 +116,14 @@ export const checkout = async (opts: CheckoutOptions = {}): Promise => { const target = opts.path ?? defaultCwd; const id = joinId(scope.path, "checkout"); - for (const source of checkoutSources) { - const plan = await source(run, opts, target, id); - - if (!plan) { - continue; - } + const local = localRepo(run); + // Another repository, or an explicit ref, is cloned even over a local run. + if (local && targetsRunRepo(run, opts)) { const machine = await ensureMachine(scope); - await ciRun(run, plan.spec, () => { - return plan.fill(machine); + await ciRun(run, steps.uploadWorkingTree(id), () => { + return uploadWorkingTree(run, machine, local.path, target); }); scope.cwd ??= target; @@ -135,66 +131,22 @@ export const checkout = async (opts: CheckoutOptions = {}): Promise => { return; } - throw new CiUsageError( - '`checkout()` needs a repository. This run\'s trigger doesn\'t have one, so pass `checkout({ repo: "owner/name" })` or set `repo: "owner/name"` on the pipeline.', - ); -}; - -/** The checkout step, and how it fills the job's machine. */ -interface CheckoutPlan { - spec: StepSpec; - fill: (machine: MachineHandle) => Promise; -} - -/** - * A place `checkout()` can get files from. It returns the plan for a checkout - * it serves and `undefined` for one it doesn't. It runs outside the step, so - * it's where anything a step can't do for itself, like resolving a ref, goes. - * - * A new source, such as a tarball URL, is one more entry in `checkoutSources`. - */ -type CheckoutSource = ( - run: CiRunScope, - opts: CheckoutOptions, - target: string, - id: string, -) => Promise; - -/** The working tree, for the run's own repository when it's run locally. */ -const localTree: CheckoutSource = async (run, opts, target, id) => { - const local = localRepo(run); - - // Another repository, or an explicit ref, is cloned even over a local run. - if (!local || !targetsRunRepo(run, opts)) { - return undefined; - } - - return { - spec: steps.uploadWorkingTree(id), - fill: (machine) => { - return uploadWorkingTree(run, machine, local.path, target); - }, - }; -}; - -/** A clone from GitHub, of the run's repository or another. */ -const githubClone: CheckoutSource = async (run, opts, target, id) => { if (!opts.repo && !run.repo) { - return undefined; + throw new CiUsageError( + '`checkout()` needs a repository. This run\'s trigger doesn\'t have one, so pass `checkout({ repo: "owner/name" })` or set `repo: "owner/name"` on the pipeline.', + ); } + // Resolved outside the step, since a step can't start another. const source = await resolveSource(run, opts); + const machine = await ensureMachine(scope); - return { - spec: steps.cloneRepository(id), - fill: (machine) => { - return cloneFromGithub(run, machine, source, opts, target); - }, - }; -}; + await ciRun(run, steps.cloneRepository(id), () => { + return cloneFromGithub(run, machine, source, opts, target); + }); -/** In order: the first that serves a checkout does. */ -const checkoutSources: CheckoutSource[] = [localTree, githubClone]; + scope.cwd ??= target; +}; const getSandbox = async (run: CiRunScope, machine: MachineHandle) => { const sandbox = await run.ci.client.sandboxes.get(machine.id); From cdf6c9615f8e84362646d3a22997a659aea123b4 Mon Sep 17 00:00:00 2001 From: Jack Williams Date: Tue, 13 Oct 2026 23:51:05 +0000 Subject: [PATCH 15/18] refactor(ci): resolve the default branch inside ref resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The commit lookup accepts HEAD, so a repository with no ref resolves in its one ref step. This drops the separate default-branch step, its run-scope map and its step name. Step IDs for a repo without a ref are now `github › ref:owner/name@HEAD`; the branch is unreleased, so no in-flight run depends on the old IDs. Co-Authored-By: Claude Opus 5.5 --- packages/ci/src/github/source.test.ts | 52 ++++++++++++++++----------- packages/ci/src/github/source.ts | 45 ++++------------------- packages/ci/src/pipeline/names.ts | 9 ----- packages/ci/src/pipeline/pipeline.ts | 1 - packages/ci/src/pipeline/scope.ts | 6 ---- 5 files changed, 39 insertions(+), 74 deletions(-) diff --git a/packages/ci/src/github/source.test.ts b/packages/ci/src/github/source.test.ts index 26a56388c..8c56de62a 100644 --- a/packages/ci/src/github/source.test.ts +++ b/packages/ci/src/github/source.test.ts @@ -57,19 +57,28 @@ const setup = ( return { api, gh, ci }; }; -/** Make `acme/platform`'s `main` branch resolve to a commit. */ +/** Make `acme/platform`'s default branch and `main` and `v2` resolve to commits. */ const platform = (gh: FakeGitHub, sha = "cafe1234") => { - gh.route("GET /repos/acme/platform", { default_branch: "main" }); + gh.route("GET /repos/acme/platform/commits/HEAD", { sha }); gh.route("GET /repos/acme/platform/commits/main", { sha }); gh.route("GET /repos/acme/platform/commits/v2", { sha: "beef5678" }); }; +/** Make a repository look missing to whoever asks, as GitHub does for no access. */ +const hidden = (gh: FakeGitHub, fullName: string) => { + gh.route(`GET /repos/${fullName}`, { message: "Not Found" }, 404); + + gh.route( + `GET /repos/${fullName}/commits/HEAD`, + { message: "Not Found" }, + 404, + ); +}; + /** Make `ref` of the run's own repository resolve to a commit. */ const ownRepo = (gh: FakeGitHub, ref: string, sha: string) => { - gh.route("GET /repos/inngest/inngest-js", { default_branch: "main" }); - gh.route(`GET /repos/inngest/inngest-js/commits/${ref}`, { sha }); }; @@ -162,8 +171,7 @@ describe("checkout() of another repository", () => { const result = await checkingOut(ci, { build: { repo: "acme/platform" } }); expect(result.type).toBe("function-resolved"); - expect(result.stepIds).toContain("github › default-branch:acme/platform"); - expect(result.stepIds).toContain("github › ref:acme/platform@main"); + expect(result.stepIds).toContain("github › ref:acme/platform@HEAD"); expect(result.stepIds).toContain("build › checkout"); expect(cloneScripts(api)).toHaveLength(1); @@ -209,21 +217,21 @@ describe("checkout() of another repository", () => { expect(cloneScripts(api)).toHaveLength(2); }); - test("repo alone and repo with the default branch share one resolve step", async () => { + test("repo alone resolves the default branch once per run", async () => { const { gh, ci } = setup(); platform(gh); const result = await checkingOut(ci, { first: { repo: "acme/platform" }, - second: { repo: "acme/platform", ref: "main", path: "/other" }, + second: { repo: "acme/platform", path: "/other" }, }); expect(result.type).toBe("function-resolved"); expect(refSteps(result.stepIds)).toEqual([ - "github › ref:acme/platform@main", + "github › ref:acme/platform@HEAD", ]); - expect(requestsTo(gh, "/repos/acme/platform/commits/main")).toHaveLength(1); + expect(requestsTo(gh, "/repos/acme/platform/commits/HEAD")).toHaveLength(1); expect(result.steps["first › checkout"]).toMatchObject({ sha: "cafe1234" }); expect(result.steps["second › checkout"]).toMatchObject({ sha: "cafe1234", @@ -336,6 +344,7 @@ describe("what a failed lookup says", () => { { name: "no default branch says so, not undefined", routes: (gh) => { + gh.route("GET /repos/acme/platform/commits/HEAD", {}, 404); gh.route("GET /repos/acme/platform", {}); }, contains: ["the default branch"], @@ -347,7 +356,7 @@ describe("what a failed lookup says", () => { platform(gh); gh.route( - "GET /repos/acme/platform/commits/main", + "GET /repos/acme/platform/commits/HEAD", { message: "Git Repository is empty." }, 409, ); @@ -357,7 +366,7 @@ describe("what a failed lookup says", () => { { name: "the token provider says the token, not the GitHub App", routes: (gh) => { - gh.route("GET /repos/acme/platform", { message: "Not Found" }, 404); + hidden(gh, "acme/platform"); }, contains: ["The token can't access"], excludes: ["GitHub App"], @@ -517,7 +526,7 @@ describe("the GitHub App's installation", () => { onlyInstallation99(gh, "/repos/acme/platform", { default_branch: "main" }); - onlyInstallation99(gh, "/repos/acme/platform/commits/main", { + onlyInstallation99(gh, "/repos/acme/platform/commits/HEAD", { sha: "cafe1234", }); @@ -531,7 +540,7 @@ describe("the GitHub App's installation", () => { expect(paths).toContain("GET /repos/acme/platform/installation"); expect(paths).toContain("POST /app/installations/99/access_tokens"); - expect(result.steps["github › ref:acme/platform@main"]).toMatchObject({ + expect(result.steps["github › ref:acme/platform@HEAD"]).toMatchObject({ installationId: 99, sha: "cafe1234", }); @@ -568,7 +577,7 @@ describe("the GitHub App's installation", () => { tokenRoutes(gh); - gh.route("GET /repos/acme/secret", { message: "Not Found" }, 404); + hidden(gh, "acme/secret"); gh.route( "GET /repos/acme/secret/installation", @@ -594,7 +603,10 @@ describe("the GitHub App's installation", () => { tokenRoutes(gh); - gh.handle("GET /repos/acme/platform", rateLimited({ "retry-after": "1" })); + gh.handle( + "GET /repos/acme/platform/commits/HEAD", + rateLimited({ "retry-after": "1" }), + ); const result = await checkoutRepo(ci); @@ -604,7 +616,7 @@ describe("the GitHub App's installation", () => { expect( requestsOf(gh).filter((path) => { - return path === "GET /repos/acme/platform"; + return path === "GET /repos/acme/platform/commits/HEAD"; }).length, ).toBeGreaterThan(1); }); @@ -614,7 +626,7 @@ describe("the GitHub App's installation", () => { tokenRoutes(gh); - gh.route("GET /repos/acme/platform", { message: "Not Found" }, 404); + hidden(gh, "acme/platform"); gh.handle("GET /repos/acme/platform/installation", rateLimited({})); @@ -630,7 +642,7 @@ describe("the GitHub App's installation", () => { tokenRoutes(gh); - gh.route("GET /repos/acme/platform", { message: "Not Found" }, 404); + hidden(gh, "acme/platform"); gh.route( "GET /repos/acme/platform/installation", @@ -714,7 +726,7 @@ describe("files() of another repository or ref", () => { test("defaults the ref to the repository's default branch", async () => { const { paths } = await cachedNames("blob1"); - expect(paths).toContain("/repos/acme/platform/commits/main"); + expect(paths).toContain("/repos/acme/platform/commits/HEAD"); }); test("a truncated tree throws instead of hashing part of the files", async () => { diff --git a/packages/ci/src/github/source.ts b/packages/ci/src/github/source.ts index 9371ed733..0807576c5 100644 --- a/packages/ci/src/github/source.ts +++ b/packages/ci/src/github/source.ts @@ -199,17 +199,17 @@ const locate = async ( } }; -/** The commit a ref points at, in a repository this client can read. */ +/** The commit a ref (default: the default branch) points at, in a repository this client can read. */ const readCommit = async ( octokit: Octokit, source: ResolvedSource, - ref: string, + ref: string | undefined, ): Promise => { try { const { data } = await octokit.rest.repos.getCommit({ owner: source.owner, repo: source.name, - ref, + ref: ref ?? "HEAD", }); return data.sha; @@ -236,45 +236,14 @@ const readCommit = async ( } }; -/** - * The default branch's name, from one step per repository per run. `repo` and - * `ref: ""` then share one commit step, keyed on the branch. - */ -const defaultBranch = (run: CiRunScope, fullName: string): Promise => { - return once(run.defaultBranches, fullName.toLowerCase(), () => { - return inGitHubSpan(run, () => { - return ciRun(run, steps.resolveDefaultBranch(fullName), async () => { - const { value } = await locate( - run, - fullName, - async (octokit, source) => { - const { data } = await octokit.rest.repos.get({ - owner: source.owner, - repo: source.name, - }); - - return data.default_branch; - }, - ); - - if (!value) { - throw notFoundError(fullName, undefined); - } - - return value; - }); - }); - }); -}; - /** * Resolve what a caller asked to read to a commit. * * The run's own repository and commit come straight from the run. Anything * else is looked up in memoized steps, in the run's GitHub span rather than * whichever job asked first, so every replay and every job sees the same - * commit: one step finds a repository's default branch when no `ref` is given, - * and one step per repository and branch finds the commit. Call it outside a + * commit: one step per repository and ref finds the commit, where no `ref` means + * the repository's default branch. Call it outside a * step: a step can't start another. Called again for what it already resolved, * it starts nothing, so a step can call it. */ @@ -306,8 +275,8 @@ export const resolveSource = async ( // and not something a step retries. providerOf(run); - const ref = spec.ref ?? (await defaultBranch(run, fullName)); - const key = sourceKey(fullName, ref); + const { ref } = spec; + const key = sourceKey(fullName, ref ?? "HEAD"); return once(run.sources, key, () => { return inGitHubSpan(run, () => { diff --git a/packages/ci/src/pipeline/names.ts b/packages/ci/src/pipeline/names.ts index 2446f2039..8e6e9cad2 100644 --- a/packages/ci/src/pipeline/names.ts +++ b/packages/ci/src/pipeline/names.ts @@ -261,15 +261,6 @@ export const steps = { }; }, - resolveDefaultBranch: (fullName: string): StepSpec => { - return { - id: boundedName( - joinId("github", `default-branch:${fullName.toLowerCase()}`), - ), - name: "Resolve ref", - }; - }, - resolveRef: (key: string): StepSpec => { return { id: boundedName(joinId("github", `ref:${key}`)), diff --git a/packages/ci/src/pipeline/pipeline.ts b/packages/ci/src/pipeline/pipeline.ts index c42184a4e..fc6bb28ce 100644 --- a/packages/ci/src/pipeline/pipeline.ts +++ b/packages/ci/src/pipeline/pipeline.ts @@ -223,7 +223,6 @@ const newRunScope = ({ ...(build ? { build } : {}), builds: new Map(), sources: new Map(), - defaultBranches: new Map(), jobCalls: new Map(), createdSnapshots: new Set(), summaries: [], diff --git a/packages/ci/src/pipeline/scope.ts b/packages/ci/src/pipeline/scope.ts index 5663693a8..55aa60c42 100644 --- a/packages/ci/src/pipeline/scope.ts +++ b/packages/ci/src/pipeline/scope.ts @@ -201,12 +201,6 @@ export interface CiRunScope { * run. It holds promises and nothing else. */ sources: Map>; - /** - * The default branch of each repository `checkout()` and `files()` have - * asked for without a `ref`, keyed by repository, so a repository's default - * branch is found by one step per run. It holds promises and nothing else. - */ - defaultBranches: Map>; /** How many direct calls of each job have started, keyed by job ID. */ jobCalls: Map; /** From f75f9733cf3eac02745f365de4eaea748d692843 Mon Sep 17 00:00:00 2001 From: Jack Williams Date: Tue, 13 Oct 2026 23:53:23 +0000 Subject: [PATCH 16/18] refactor(ci): table the repeated source tests Turn the repeated checkout, installation, local-run and files() cases in source.test.ts into test.each tables and merge near-duplicate tests. The same behaviours are covered. fakeGitHub.handle stays: the installation fallback test replies by Authorization header. Co-Authored-By: Claude Opus 5.5 --- packages/ci/src/github/source.test.ts | 475 +++++++++++--------------- 1 file changed, 207 insertions(+), 268 deletions(-) diff --git a/packages/ci/src/github/source.test.ts b/packages/ci/src/github/source.test.ts index 8c56de62a..801480f4c 100644 --- a/packages/ci/src/github/source.test.ts +++ b/packages/ci/src/github/source.test.ts @@ -100,33 +100,13 @@ type Checkout = Parameters[0]; */ const checkingOut = ( ci: Ci, - jobs: Record, + opts: Record, event: typeof prEvent | typeof nightly = prEvent, ) => { - return running( - ci, - Object.fromEntries( - Object.entries(jobs).map(([id, opts]) => { - return [ - id, - async () => { - await checkout(opts); - }, - ]; - }), - ), - event, - ); -}; - -/** Run jobs side by side in a pull request run or a cron. */ -const running = ( - ci: Ci, - bodies: Record Promise>, - event: typeof prEvent | typeof nightly = prEvent, -) => { - const jobs = Object.entries(bodies).map(([id, body]) => { - return ci.job(id, body); + const jobs = Object.entries(opts).map(([id, each]) => { + return ci.job(id, async () => { + await checkout(each); + }); }); const options = @@ -163,7 +143,7 @@ const requestsTo = (gh: FakeGitHub, path: string) => { }; describe("checkout() of another repository", () => { - test("clones the commit the default branch resolves to", async () => { + test("clones the commit the default branch resolves to, without the token in any step", async () => { const { api, gh, ci } = setup(); platform(gh); @@ -176,81 +156,69 @@ describe("checkout() of another repository", () => { expect(cloneScripts(api)).toHaveLength(1); expect(cloneScripts(api)[0]).toContain("checkout 'cafe1234'"); - expect(cloneScripts(api)[0]).not.toContain(secret); expect(result.steps["build › checkout"]).toMatchObject({ sha: "cafe1234", source: "github", }); - }); - - test("keeps the installation token out of every step", async () => { - const { api, gh, ci } = setup(); - - platform(gh); - - const result = await checkingOut(ci, { - build: { repo: "acme/platform", ref: "v2" }, - }); - expect(result.type).toBe("function-resolved"); - expect(JSON.stringify(result.steps)).not.toContain(secret); - expect(JSON.stringify(result.metadata)).not.toContain(secret); - expect(JSON.stringify(api.commands)).not.toContain(secret); - }); - - test("resolves a ref to a commit once per run", async () => { - const { api, gh, ci } = setup(); - - platform(gh); - - const result = await checkingOut(ci, { - first: { repo: "acme/platform", ref: "main" }, - second: { repo: "acme/platform", ref: "main", path: "/other" }, - }); - - expect(result.type).toBe("function-resolved"); - expect(refSteps(result.stepIds)).toEqual([ - "github › ref:acme/platform@main", - ]); - expect(requestsTo(gh, "/repos/acme/platform/commits/main")).toHaveLength(1); - expect(cloneScripts(api)).toHaveLength(2); + for (const seen of [result.steps, result.metadata, api.commands]) { + expect(JSON.stringify(seen)).not.toContain(secret); + } }); - test("repo alone resolves the default branch once per run", async () => { - const { gh, ci } = setup(); - - platform(gh); - - const result = await checkingOut(ci, { - first: { repo: "acme/platform" }, - second: { repo: "acme/platform", path: "/other" }, - }); + test.each([ + { ref: "main", step: "main", path: "commits/main" }, + { ref: undefined, step: "HEAD", path: "commits/HEAD" }, + ])( + "resolves ref $ref to a commit once per run", + async ({ ref, step, path }) => { + const { api, gh, ci } = setup(); + + platform(gh); + + const opts = { repo: "acme/platform", ...(ref ? { ref } : {}) }; + + const result = await checkingOut(ci, { + first: opts, + second: { ...opts, path: "/other" }, + }); + + expect(result.type).toBe("function-resolved"); + expect(refSteps(result.stepIds)).toEqual([ + `github › ref:acme/platform@${step}`, + ]); + expect(requestsTo(gh, `/repos/acme/platform/${path}`)).toHaveLength(1); + expect(cloneScripts(api)).toHaveLength(2); + }, + ); - expect(result.type).toBe("function-resolved"); - expect(refSteps(result.stepIds)).toEqual([ - "github › ref:acme/platform@HEAD", - ]); - expect(requestsTo(gh, "/repos/acme/platform/commits/HEAD")).toHaveLength(1); - expect(result.steps["first › checkout"]).toMatchObject({ sha: "cafe1234" }); - expect(result.steps["second › checkout"]).toMatchObject({ + test.each([ + { + name: "works when the run's trigger has no repository", + routes: platform, + opts: { repo: "acme/platform", ref: "main" }, + event: nightly, sha: "cafe1234", - }); - }); - - test("works when the run's trigger has no repository", async () => { + }, + { + name: "a ref alone overrides the run's commit for its own repository", + routes: (gh: FakeGitHub) => { + ownRepo(gh, "release", "feed9999"); + }, + opts: { ref: "release" }, + event: prEvent, + sha: "feed9999", + }, + ])("$name", async ({ routes, opts, event, sha }) => { const { api, gh, ci } = setup(); - platform(gh); + routes(gh); - const result = await checkingOut( - ci, - { build: { repo: "acme/platform", ref: "main" } }, - nightly, - ); + const result = await checkingOut(ci, { build: opts }, event); expect(result.type).toBe("function-resolved"); - expect(cloneScripts(api)[0]).toContain("checkout 'cafe1234'"); + expect(cloneScripts(api)[0]).toContain(`checkout '${sha}'`); }); test("still needs a repository when none is given", async () => { @@ -262,17 +230,6 @@ describe("checkout() of another repository", () => { expect(JSON.stringify(result.error)).toContain("needs a repository"); }); - test("a ref alone overrides the run's commit for its own repository", async () => { - const { api, gh, ci } = setup(); - - ownRepo(gh, "release", "feed9999"); - - const result = await checkingOut(ci, { build: { ref: "release" } }); - - expect(result.type).toBe("function-resolved"); - expect(cloneScripts(api)[0]).toContain("checkout 'feed9999'"); - }); - test("a long ref keeps the step ID bounded", async () => { const { gh, ci } = setup(); const ref = "x".repeat(600); @@ -455,15 +412,6 @@ describe("the GitHub App's installation", () => { }); }; - /** Answer as the other installation would, and 404 for the run's own. */ - const onlyInstallation99 = (gh: FakeGitHub, path: string, body: unknown) => { - gh.handle(`GET ${path}`, (request) => { - return request.authorization?.includes("tok_run") - ? { body: { message: "Not Found" }, status: 404 } - : { body }; - }); - }; - const rateLimited = (headers: Record) => { return () => { return { @@ -478,7 +426,7 @@ describe("the GitHub App's installation", () => { return checkingOut(ci, { build: { repo } }); }; - test("the run's own installation is used first", async () => { + test("the run's own installation is used first, with a token narrowed to the repository", async () => { const { api, gh, ci } = setup(app); platform(gh); @@ -494,41 +442,45 @@ describe("the GitHub App's installation", () => { expect(paths).toContain("POST /app/installations/1/access_tokens"); expect(paths).not.toContain("GET /repos/acme/platform/installation"); expect(paths).not.toContain("POST /app/installations/99/access_tokens"); - }); - - test("tokens are narrowed to the one repository", async () => { - const { gh, ci } = setup(app); - platform(gh); - tokenRoutes(gh); - - await checkoutRepo(ci); - - const minted = gh.requests.filter((request) => { - return ( - request.method === "POST" && request.path.endsWith("/access_tokens") - ); - }); - - expect(minted.length).toBeGreaterThan(0); - - for (const request of minted) { - expect(request.body).toMatchObject({ repositories: ["platform"] }); + for (const request of gh.requests) { + if (request.path.endsWith("/access_tokens")) { + expect(request.body).toMatchObject({ repositories: ["platform"] }); + } } }); - test("is found from the repository when the run's can't see it", async () => { + test.each([ + { + name: "the run's installation can't see the repository", + routes: (gh: FakeGitHub) => { + gh.handle("GET /repos/acme/platform*", (request) => { + return request.authorization?.includes("tok_run") + ? { body: { message: "Not Found" }, status: 404 } + : { body: { sha: "cafe1234" } }; + }); + }, + }, + { + name: "the narrowed token is refused", + routes: (gh: FakeGitHub) => { + platform(gh); + + gh.route( + "POST /app/installations/1/access_tokens", + { message: "Validation Failed" }, + 422, + ); + }, + }, + ])("is found from the repository when $name", async ({ routes }) => { const { api, gh, ci } = setup(app); tokenRoutes(gh); - gh.route("GET /repos/acme/platform/installation", { id: 99 }); + routes(gh); - onlyInstallation99(gh, "/repos/acme/platform", { default_branch: "main" }); - - onlyInstallation99(gh, "/repos/acme/platform/commits/HEAD", { - sha: "cafe1234", - }); + gh.route("GET /repos/acme/platform/installation", { id: 99 }); const result = await checkoutRepo(ci); @@ -548,114 +500,93 @@ describe("the GitHub App's installation", () => { expect(JSON.stringify(result.steps)).not.toContain(secret); }); - test("is found from the repository when the narrowed token is refused", async () => { - const { api, gh, ci } = setup(app); - - platform(gh); - tokenRoutes(gh); - - gh.route( - "POST /app/installations/1/access_tokens", - { message: "Validation Failed" }, - 422, - ); - - gh.route("GET /repos/acme/platform/installation", { id: 99 }); - - const result = await checkoutRepo(ci); - - expect(result.type).toBe("function-resolved"); - expect(cloneScripts(api)[0]).toContain("checkout 'cafe1234'"); - - expect(requestsOf(gh)).toContain( - "POST /app/installations/99/access_tokens", - ); - }); - - test("an app that can't access the repository says so", async () => { - const { api, gh, ci } = setup(app); - - tokenRoutes(gh); - - hidden(gh, "acme/secret"); - - gh.route( - "GET /repos/acme/secret/installation", - { message: "Not Found" }, - 404, - ); - - const result = await checkoutRepo(ci, "acme/secret"); - - expect(result.type).toBe("function-rejected"); - expect(result.retriable).toBe(false); - - const message = JSON.stringify(result.error); - - expect(message).toContain("acme/secret"); - expect(message).toContain("GitHub App"); - - expect(cloneScripts(api)).toHaveLength(0); - }); - - test("a rate limit retries instead of reading as no access", async () => { - const { gh, ci } = setup(app); - - tokenRoutes(gh); - - gh.handle( - "GET /repos/acme/platform/commits/HEAD", - rateLimited({ "retry-after": "1" }), - ); - - const result = await checkoutRepo(ci); - - expect(result.type).toBe("function-rejected"); - expect(JSON.stringify(result.error)).toContain("rate limit"); - expect(JSON.stringify(result.error)).not.toContain("can't access"); - - expect( - requestsOf(gh).filter((path) => { - return path === "GET /repos/acme/platform/commits/HEAD"; - }).length, - ).toBeGreaterThan(1); - }); + test.each([ + { + name: "an app that can't access the repository says so", + repo: "acme/secret", + routes: (gh: FakeGitHub) => { + hidden(gh, "acme/secret"); - test("a rate limit looking up the installation retries too", async () => { - const { gh, ci } = setup(app); + gh.route( + "GET /repos/acme/secret/installation", + { message: "Not Found" }, + 404, + ); + }, + retriable: false, + contains: ["acme/secret", "GitHub App"], + }, + { + name: "a rate limit reading the repository retries instead of reading as no access", + routes: (gh: FakeGitHub) => { + gh.handle( + "GET /repos/acme/platform/commits/HEAD", + rateLimited({ "retry-after": "1" }), + ); + }, + contains: ["rate limit"], + excludes: ["can't access"], + }, + { + name: "a rate limit looking up the installation retries too", + routes: (gh: FakeGitHub) => { + hidden(gh, "acme/platform"); - tokenRoutes(gh); + gh.handle("GET /repos/acme/platform/installation", rateLimited({})); + }, + contains: ["rate limit"], + excludes: ["can't access"], + }, + { + name: "a rejected App says its credentials are the problem", + routes: (gh: FakeGitHub) => { + hidden(gh, "acme/platform"); - hidden(gh, "acme/platform"); + gh.route( + "GET /repos/acme/platform/installation", + { message: "Bad credentials" }, + 401, + ); + }, + retriable: false, + contains: ["credentials"], + }, + ])( + "$name", + async ({ + repo = "acme/platform", + routes, + retriable, + contains, + excludes = [], + }) => { + const { api, gh, ci } = setup(app); - gh.handle("GET /repos/acme/platform/installation", rateLimited({})); + tokenRoutes(gh); - const result = await checkoutRepo(ci); + routes(gh); - expect(result.type).toBe("function-rejected"); - expect(JSON.stringify(result.error)).toContain("rate limit"); - expect(JSON.stringify(result.error)).not.toContain("can't access"); - }); + const result = await checkoutRepo(ci, repo); - test("a rejected App says its credentials are the problem", async () => { - const { gh, ci } = setup(app); + expect(result.type).toBe("function-rejected"); - tokenRoutes(gh); + if (retriable !== undefined) { + expect(result.retriable).toBe(retriable); + } - hidden(gh, "acme/platform"); + const message = JSON.stringify(result.error); - gh.route( - "GET /repos/acme/platform/installation", - { message: "Bad credentials" }, - 401, - ); + for (const text of contains) { + expect(message).toContain(text); + } - const result = await checkoutRepo(ci); + for (const text of excludes) { + expect(message).not.toContain(text); + } - expect(result.type).toBe("function-rejected"); - expect(result.retriable).toBe(false); - expect(JSON.stringify(result.error)).toContain("credentials"); - }); + expect(cloneScripts(api)).toHaveLength(0); + }, + ); }); describe("files() of another repository or ref", () => { @@ -706,12 +637,19 @@ describe("files() of another repository or ref", () => { }; }; - test("reads the tree of the resolved commit, not the ref name", async () => { - const { paths } = await cachedNames("blob1", "main"); - - expect(paths).toContain("/repos/acme/platform/git/trees/cafe1234"); - expect(paths).not.toContain("/repos/acme/platform/git/trees/main"); - }); + test.each([ + { ref: "main", resolved: "commits/main" }, + { ref: undefined, resolved: "commits/HEAD" }, + ])( + "reads the tree of the commit ref $ref resolves to, not the ref name", + async ({ ref, resolved }) => { + const { paths } = await cachedNames("blob1", ref); + + expect(paths).toContain(`/repos/acme/platform/${resolved}`); + expect(paths).toContain("/repos/acme/platform/git/trees/cafe1234"); + expect(paths).not.toContain("/repos/acme/platform/git/trees/main"); + }, + ); test("the key changes when the commit's matched files change", async () => { const before = await cachedNames("blob1", "main"); @@ -723,12 +661,6 @@ describe("files() of another repository or ref", () => { expect(after.names).not.toEqual(before.names); }); - test("defaults the ref to the repository's default branch", async () => { - const { paths } = await cachedNames("blob1"); - - expect(paths).toContain("/repos/acme/platform/commits/HEAD"); - }); - test("a truncated tree throws instead of hashing part of the files", async () => { const { gh, ci } = setup(); @@ -750,19 +682,22 @@ describe("files() of another repository or ref", () => { expect(message).toContain("narrower patterns"); }); - test.each(["acme", "acme/platform/extra", "acme/..", "ac me/platform"])( - "rejects the repo %j when the part is defined", - (repo) => { - expect(() => { - return files("a", { repo }); - }).toThrow(CiUsageError); - }, - ); + test.each([ + ["acme", false], + ["acme/platform/extra", false], + ["acme/..", false], + ["ac me/platform", false], + ["octo_cat/platform", true], + ])("the part's repo %j is valid: %s", (repo, valid) => { + const define = () => { + return files("a", { repo }); + }; - test("accepts an Enterprise Managed User owner", () => { - expect(() => { - return files("a", { repo: "octo_cat/platform" }); - }).not.toThrow(); + if (valid) { + expect(define).not.toThrow(); + } else { + expect(define).toThrow(CiUsageError); + } }); test("keeps the part's own options", () => { @@ -833,25 +768,29 @@ describe("a local run", () => { expect(gh.requests).toHaveLength(0); }); - test("clones from GitHub for another repository", async () => { - const { api, gh, ci } = setup(); - - platform(gh); - - const result = await runLocal(ci, [{ repo: "acme/platform" }]); - - expect(result.type).toBe("function-resolved"); - expect(cloneScripts(api)[0]).toContain("checkout 'cafe1234'"); - }); - - test("clones from GitHub for an explicit ref", async () => { + test.each([ + { + name: "another repository", + routes: platform, + opts: { repo: "acme/platform" }, + sha: "cafe1234", + }, + { + name: "an explicit ref", + routes: (gh: FakeGitHub) => { + ownRepo(gh, "release", "feed9999"); + }, + opts: { ref: "release" }, + sha: "feed9999", + }, + ])("clones from GitHub for $name", async ({ routes, opts, sha }) => { const { api, gh, ci } = setup(); - ownRepo(gh, "release", "feed9999"); + routes(gh); - const result = await runLocal(ci, [{ ref: "release" }]); + const result = await runLocal(ci, [opts]); expect(result.type).toBe("function-resolved"); - expect(cloneScripts(api)[0]).toContain("checkout 'feed9999'"); + expect(cloneScripts(api)[0]).toContain(`checkout '${sha}'`); }); }); From 22a0510f5623579c11e1a892fc18abd51f2fdf8c Mon Sep 17 00:00:00 2001 From: Jack Williams Date: Wed, 14 Oct 2026 01:31:15 +0000 Subject: [PATCH 17/18] fix(ci): hand builds the invoker's resolved sources; reject empty repo/ref in files() Co-Authored-By: Claude Opus 5.5 --- packages/ci/src/cache/cache.ts | 6 +++--- packages/ci/src/github/source.test.ts | 23 +++++++++++++++++++++++ packages/ci/src/github/source.ts | 23 +++++++++++++++++++++++ packages/ci/src/pipeline/cacheBuild.ts | 3 +++ packages/ci/src/pipeline/job.ts | 2 ++ packages/ci/src/pipeline/pipeline.ts | 6 +++++- 6 files changed, 59 insertions(+), 4 deletions(-) diff --git a/packages/ci/src/cache/cache.ts b/packages/ci/src/cache/cache.ts index b8a3487eb..b5cf148bc 100644 --- a/packages/ci/src/cache/cache.ts +++ b/packages/ci/src/cache/cache.ts @@ -816,7 +816,7 @@ export const files = ( const options = typeof last === "object" ? last : undefined; // Checked here, where the pipeline is defined, so a bad `repo` fails at boot. - if (options?.repo) { + if (options?.repo !== undefined) { parseRepo(options.repo); } @@ -828,7 +828,7 @@ export const files = ( kind: "inngest/ci.cacheKeyPart", type: "files", patterns, - ...(options?.repo ? { repo: options.repo } : {}), - ...(options?.ref ? { ref: options.ref } : {}), + ...(options?.repo === undefined ? {} : { repo: options.repo }), + ...(options?.ref === undefined ? {} : { ref: options.ref }), }; }; diff --git a/packages/ci/src/github/source.test.ts b/packages/ci/src/github/source.test.ts index 801480f4c..4f0d01c16 100644 --- a/packages/ci/src/github/source.test.ts +++ b/packages/ci/src/github/source.test.ts @@ -25,6 +25,7 @@ import { createFakeSandboxApi } from "../testing/fakeSandbox.ts"; import { runFunction } from "../testing/runFunction.ts"; import type { GitHubProvider } from "./auth.ts"; import { githubApp, githubToken } from "./auth.ts"; +import { settledSources } from "./source.ts"; type FakeGitHub = ReturnType; @@ -683,6 +684,7 @@ describe("files() of another repository or ref", () => { }); test.each([ + ["", false], ["acme", false], ["acme/platform/extra", false], ["acme/..", false], @@ -794,3 +796,24 @@ describe("a local run", () => { expect(cloneScripts(api)[0]).toContain(`checkout '${sha}'`); }); }); + +describe("the sources handed to a build run", () => { + test("keeps what resolved and leaves out what failed", async () => { + const source = { + owner: "acme", + name: "platform", + fullName: "acme/platform", + sha: "abc", + }; + const failed = Promise.reject(new Error("no access")); + + const settled = await settledSources({ + sources: new Map([ + ["acme/platform@main", Promise.resolve(source)], + ["acme/other@HEAD", failed], + ]), + } as unknown as Parameters[0]); + + expect(settled).toEqual({ "acme/platform@main": source }); + }); +}); diff --git a/packages/ci/src/github/source.ts b/packages/ci/src/github/source.ts index 0807576c5..f756a35f2 100644 --- a/packages/ci/src/github/source.ts +++ b/packages/ci/src/github/source.ts @@ -236,6 +236,29 @@ const readCommit = async ( } }; +/** + * The sources this run has resolved, which a build run is handed so it reads + * the commits the invoker keyed on even if a branch has moved since. One that + * failed or is still resolving is left out: the build resolves it itself. + */ +export const settledSources = async ( + run: CiRunScope, +): Promise> => { + const settled: Record = {}; + + await Promise.all( + [...run.sources].map(async ([key, source]) => { + try { + settled[key] = await source; + } catch { + // Left for the build to resolve. + } + }), + ); + + return settled; +}; + /** * Resolve what a caller asked to read to a commit. * diff --git a/packages/ci/src/pipeline/cacheBuild.ts b/packages/ci/src/pipeline/cacheBuild.ts index e52b66fd1..06425cfcc 100644 --- a/packages/ci/src/pipeline/cacheBuild.ts +++ b/packages/ci/src/pipeline/cacheBuild.ts @@ -18,6 +18,7 @@ import type { Inngest, InngestFunction } from "inngest"; import { NonRetriableError } from "inngest"; import { metadataMiddleware, sandboxMiddleware } from "inngest/experimental"; import type { CachedSnapshot, CacheTarget } from "../cache/cache.ts"; +import type { ResolvedSource } from "../github/source.ts"; import type { CheckAnnotation, MachineConfig, @@ -80,6 +81,8 @@ export interface CacheBuildData extends Record { base?: CacheBuildResult; /** The pipeline's repository, with the working tree's location for local runs. */ repo?: RepoContext; + /** The commits the invoking run resolved for other repositories, by source key. */ + sources?: Record; /** The invoking run's machine settings, for jobs that set none of their own. */ machine?: MachineConfig; /** diff --git a/packages/ci/src/pipeline/job.ts b/packages/ci/src/pipeline/job.ts index dd43337fb..b83a36fad 100644 --- a/packages/ci/src/pipeline/job.ts +++ b/packages/ci/src/pipeline/job.ts @@ -21,6 +21,7 @@ import { CommandTimeoutError, } from "../errors.ts"; import type { CheckReporter } from "../github/checks.ts"; +import { settledSources } from "../github/source.ts"; import { identityOf, ownJob, @@ -201,6 +202,7 @@ export const invokeBuild = async ({ ...(unnamed ? { unnamed } : {}), ...(base ? { base: baseForBuild(base) } : {}), ...(run.repo ? { repo: run.repo } : {}), + sources: await settledSources(run), ...(run.machine ? { machine: run.machine } : {}), rootRunId, parent: { diff --git a/packages/ci/src/pipeline/pipeline.ts b/packages/ci/src/pipeline/pipeline.ts index fc6bb28ce..e30fd36fb 100644 --- a/packages/ci/src/pipeline/pipeline.ts +++ b/packages/ci/src/pipeline/pipeline.ts @@ -222,7 +222,11 @@ const newRunScope = ({ ...(repo ? { repo } : {}), ...(build ? { build } : {}), builds: new Map(), - sources: new Map(), + sources: new Map( + Object.entries(build?.sources ?? {}).map(([key, source]) => { + return [key, Promise.resolve(source)]; + }), + ), jobCalls: new Map(), createdSnapshots: new Set(), summaries: [], From 9c9fedc4a10de23a04416955e948354df2cb2b49 Mon Sep 17 00:00:00 2001 From: Jack Williams Date: Wed, 14 Oct 2026 01:45:31 +0000 Subject: [PATCH 18/18] fix(ci): say a truncated tree needs a string key; test builds reuse the invoker's commits Narrower files() patterns cannot help: the whole recursive tree is fetched and rejected when GitHub truncates it. Co-Authored-By: Claude Opus 5.5 --- packages/ci/src/cache/cache.ts | 2 +- packages/ci/src/github/source.test.ts | 42 ++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/packages/ci/src/cache/cache.ts b/packages/ci/src/cache/cache.ts index b5cf148bc..70ee43485 100644 --- a/packages/ci/src/cache/cache.ts +++ b/packages/ci/src/cache/cache.ts @@ -218,7 +218,7 @@ const hashTree = ( // ignores the rest never changes when they do. if (listing.truncated) { throw new NonRetriableError( - `\`${fullName}\` is too large to hash \`files()\` from GitHub's tree API, which cut the file listing short. Use narrower patterns, or a string key.`, + `\`${fullName}\` is too large to hash \`files()\` from GitHub's tree API, which cut the file listing short. Key the job on a string instead.`, ); } diff --git a/packages/ci/src/github/source.test.ts b/packages/ci/src/github/source.test.ts index 4f0d01c16..dc25d4324 100644 --- a/packages/ci/src/github/source.test.ts +++ b/packages/ci/src/github/source.test.ts @@ -680,7 +680,7 @@ describe("files() of another repository or ref", () => { const message = JSON.stringify(result.error); expect(message).toContain("too large to hash"); - expect(message).toContain("narrower patterns"); + expect(message).toContain("Key the job on a string"); }); test.each([ @@ -816,4 +816,44 @@ describe("the sources handed to a build run", () => { expect(settled).toEqual({ "acme/platform@main": source }); }); + + test("a build reads the commit the invoker keyed on after the branch moves", async () => { + const { gh, ci } = setup(); + + let lookups = 0; + + gh.handle("GET /repos/acme/platform/commits/main", () => { + lookups++; + + return { body: { sha: lookups === 1 ? "cafe1234" : "beef5678" } }; + }); + + gh.route("GET /repos/acme/platform/git/trees/cafe1234", { + tree: [{ type: "blob", path: "a", sha: "old" }], + }); + + gh.route("GET /repos/acme/platform/git/trees/beef5678", { + tree: [{ type: "blob", path: "a", sha: "new" }], + }); + + const result = await runFunction( + ci.pipeline({ id: "pr", on: prTrigger }, async () => { + const image = ci.job( + { + id: "image", + cache: { key: files("a", { repo: "acme/platform", ref: "main" }) }, + }, + async () => { + await $`build image`; + }, + ); + + return image(); + }), + { event: prEvent }, + ); + + expect(result.type).toBe("function-resolved"); + expect(lookups).toBe(1); + }); });