Add cache.maxAge to CI

.changeset/ci-cache-max-age.md

@@ -0,0 +1,9 @@
+---
+"@inngest/ci": minor
+---
+
+Cached jobs can now expire with `cache.maxAge`: a snapshot older than it is rebuilt.
+
+```ts
+ci.job({ id: "base", cache: { key: files("images/node-base/**"), maxAge: "1d" } }, …);
+```

packages/ci/README.md

@@ -51,6 +51,20 @@ const test = ci.job({ id: "test", from: base }, async () => {
 
 A pipeline run is one trace. `lint` and `test` start from a snapshot of `base`, and a failed command runs again without rerunning the jobs that passed.
 
+## Caching
+
+A job with a `cache` key is reused until its key changes. Add `maxAge` to rebuild it on a schedule too: a snapshot older than that counts as a miss, and the new one takes its name.
+
+```ts
+const base = ci.job(
+  { id: "base", cache: { key: files("images/node-base/**"), maxAge: "1d" } },
+  async () => {
+    await checkout();
+    await $`pnpm install`;
+  },
+);
+```
+
 ## Another repository or ref
 
 `checkout()` and `files()` take `repo` and `ref`. A ref is a branch, tag or commit, and it's resolved to a commit once per run.

packages/ci/src/cache/cache.test.ts

@@ -10,7 +10,7 @@ import { describe, expect, test } from "vitest";
 
 import { deleteRunSnapshots } from "../machine/machine.ts";
 import type { CiRunScope } from "../pipeline/scope.ts";
-import { deleteSnapshot, resolveTakenName } from "./cache.ts";
+import { deleteSnapshot, resolveTakenName, usableFor } from "./cache.ts";
 
 /** Step results by ID, kept across the "replays" of one test. */
 type Memo = Map<string, unknown>;
@@ -89,12 +89,16 @@ describe("deleteSnapshot", () => {
 
 describe("resolveTakenName", () => {
   /** A run whose only snapshot holds the name, recording what is deleted. */
-  const holding = (deleted: string[]): CiRunScope => {
+  const holding = (
+    deleted: string[],
+    /** When the holder was created, in place of now. */
+    createdAt = new Date().toISOString(),
+  ): CiRunScope => {
     const holder = {
       id: "main-snapshot",
       name: "ci/main/base/k",
       status: "READY",
-      createdAt: new Date().toISOString(),
+      createdAt,
     };
 
     const snapshots = {
@@ -130,6 +134,7 @@ describe("resolveTakenName", () => {
       holding(deleted),
       "name-taken",
       "ci/main/base/k",
+      undefined,
       "main-snapshot",
     );
 
@@ -144,11 +149,71 @@ describe("resolveTakenName", () => {
       holding(deleted),
       "name-taken",
       "ci/main/base/k",
+      undefined,
       "main-snapshot",
       true,
     );
 
     expect(deleted).toEqual(["main-snapshot"]);
     expect(taken.cleared).toBe(true);
   });
+
+  test("a holder that is too old is replaced without being known to be broken", async () => {
+    const deleted: string[] = [];
+
+    const taken = await resolveTakenName(
+      holding(deleted, "2020-01-01T00:00:00.000Z"),
+      "name-taken",
+      "ci/main/base/k",
+      { key: "k", maxAge: "1m" },
+    );
+
+    expect(deleted).toEqual(["main-snapshot"]);
+    expect(taken.cleared).toBe(true);
+  });
+});
+
+describe("usableFor", () => {
+  const ago = (ms: number): string => {
+    return new Date(Date.now() - ms).toISOString();
+  };
+
+  const soon = new Date(Date.now() + 60_000).toISOString();
+
+  test.each([
+    ["ready and fresh", undefined, undefined, {}, true],
+    ["still creating", undefined, undefined, { status: "CREATING" }, false],
+    ["the excluded one", undefined, "s1", {}, false],
+    ["about to expire", undefined, undefined, { expiresAt: soon }, false],
+    ["younger than maxAge", "1h", undefined, { createdAt: ago(60_000) }, true],
+    [
+      "older than maxAge",
+      "1h",
+      undefined,
+      { createdAt: ago(7_200_000) },
+      false,
+    ],
+    ["an unreadable createdAt", "1h", undefined, { createdAt: "?" }, true],
+    [
+      "old with no maxAge",
+      undefined,
+      undefined,
+      { createdAt: ago(1e10) },
+      true,
+    ],
+  ] as const)("%s", (_name, maxAge, exclude, overrides, expected) => {
+    const usable = usableFor(
+      maxAge ? { key: "k", maxAge } : undefined,
+      exclude,
+    );
+
+    expect(
+      usable({
+        id: "s1",
+        status: "READY",
+        createdAt: ago(0),
+        ...overrides,
+      }),
+    ).toBe(expected);
+  });
 });

packages/ci/src/cache/cache.ts

@@ -32,6 +32,7 @@ import type {
   RepoContext,
 } from "../types.ts";
 import {
+  durationToMs,
   filterPaths,
   formatRelative,
   hash,
@@ -410,6 +411,46 @@ const isExpiring = (expiresAt: string | undefined): boolean => {
   return Number.isFinite(at) && at - Date.now() < expiryMarginMs;
 };
 
+/** The parts of a snapshot that decide whether a job may use it. */
+export interface SnapshotInfo {
+  id: string;
+  status: string;
+  createdAt: string;
+  expiresAt?: string;
+}
+
+/** Whether a job may use a snapshot. */
+export type Usable = (snapshot: SnapshotInfo) => boolean;
+
+/**
+ * What makes a snapshot usable: ready, not the one to avoid, not about to
+ * expire, and not older than the job's `cache.maxAge`. A new freshness rule is
+ * one more clause here. `defineJob` has already checked that `maxAge` parses.
+ */
+export const usableFor = (
+  cache: CacheConfig | undefined,
+  /** A snapshot that must not be used, though it may still hold the name. */
+  exclude?: string,
+): Usable => {
+  const maxAgeMs =
+    cache?.maxAge === undefined ? undefined : durationToMs(cache.maxAge);
+
+  return (snapshot) => {
+    const createdAt = Date.parse(snapshot.createdAt);
+    const tooOld =
+      maxAgeMs !== undefined &&
+      Number.isFinite(createdAt) &&
+      Date.now() - createdAt > maxAgeMs;
+
+    return (
+      snapshot.status === "READY" &&
+      snapshot.id !== exclude &&
+      !isExpiring(snapshot.expiresAt) &&
+      !tooOld
+    );
+  };
+};
+
 /** How long a lookup waits for a snapshot another build is still taking. */
 const readyWaitMs = 2 * 60 * 1000;
 const readyPollMs = 1000;
@@ -460,16 +501,16 @@ const waitUntilReady = async (
 
 /**
  * The newest snapshot with exactly this name, if it can be used: ready, or
- * ready after waiting for the build taking it, and not about to expire.
+ * ready after waiting for the build taking it, not about to expire, and not
+ * older than the job's max age.
  *
  * Any error is a miss, and only an exact name counts, since a server that
  * doesn't know names ignores the filter and lists every snapshot.
  */
 const findNamed = async (
   run: CiRunScope,
   name: string,
-  /** A snapshot that must not be used, though it may still hold the name. */
-  exclude?: string,
+  usable: Usable,
 ): Promise<CachedSnapshot | undefined> => {
   try {
     const page = (await snapshotsClient(run).list({ name, limit: 10 })) as {
@@ -480,7 +521,8 @@ const findNamed = async (
       return snapshot.name === name;
     });
 
-    if (!newest || newest.id === exclude) {
+    // Waiting is only worth it for a snapshot that would be usable once ready.
+    if (!newest || !usable({ ...newest, status: "READY" })) {
       return undefined;
     }
 
@@ -489,7 +531,7 @@ const findNamed = async (
         ? await waitUntilReady(run, newest.id)
         : newest;
 
-    if (ready?.status !== "READY" || isExpiring(ready.expiresAt)) {
+    if (!ready || !usable(ready)) {
       return undefined;
     }
 
@@ -507,11 +549,13 @@ const findInScopes = async (
   ownKey: string,
   exclude?: string,
 ): Promise<CachedSnapshot | undefined> => {
+  const usable = usableFor(cache, exclude);
+
   for (const scope of cacheScopes(run.repo, cache.scope).read) {
     const found = await findNamed(
       run,
       formatName({ kind: "cache", scope, jobId, ownKey }),
-      exclude,
+      usable,
     );
 
     if (found) {
@@ -570,7 +614,7 @@ const findCached = (
 ): Promise<CachedSnapshot | undefined> => {
   return cache
     ? findInScopes(run, jobId, cache, target.ownKey, exclude)
-    : findNamed(run, target.name, exclude);
+    : findNamed(run, target.name, usableFor(undefined, exclude));
 };
 
 /**
@@ -606,13 +650,16 @@ export const lookupBeforeBuild = async (
 
 /**
  * Find the snapshot that holds a name a build couldn't take, as a memoized
- * step. If it can't be used, because it is about to expire or is the bad one
- * the build replaces, it is deleted so the build can take the name after all.
+ * step. If it can't be used, because it is about to expire, is older than the
+ * job's max age or is the bad one the build replaces, it is deleted so the
+ * build can take the name after all.
  */
 export const resolveTakenName = async (
   run: CiRunScope,
   stepId: string,
   name: string,
+  /** The job's `cache`, whose `maxAge` a holder may be past. */
+  cache?: CacheConfig,
   exclude?: string,
   /**
    * Whether `exclude` was decided to be broken, which is the only case where
@@ -625,7 +672,7 @@ export const resolveTakenName = async (
     run,
     steps.resolveCacheName(stepId),
     async (): Promise<{ winner?: CachedSnapshot; cleared: boolean }> => {
-      const winner = await findNamed(run, name, exclude);
+      const winner = await findNamed(run, name, usableFor(cache, exclude));
 
       if (winner) {
         return { winner, cleared: false };
@@ -638,9 +685,10 @@ export const resolveTakenName = async (
           items: SnapshotResource[];
         };
 
+        const usable = usableFor(cache);
+
         for (const holder of page.items) {
-          const unusable =
-            (broken && holder.id === exclude) || isExpiring(holder.expiresAt);
+          const unusable = (broken && holder.id === exclude) || !usable(holder);
 
           if (holder.name !== name || holder.status !== "READY" || !unusable) {
             continue;

packages/ci/src/machine/machine.ts

@@ -25,7 +25,7 @@ import {
   scopeSeparator,
 } from "../pipeline/scope.ts";
 import { inSpan } from "../pipeline/spans.ts";
-import type { MachineConfig } from "../types.ts";
+import type { CacheConfig, MachineConfig } from "../types.ts";
 import { boundedName, errorMessage, isSandboxNotFound, slug } from "../util.ts";
 import { deleteSnapshots, listSandboxes } from "./admin.ts";
 
@@ -466,6 +466,8 @@ export interface SnapshotCache {
    * failing to name it isn't worth a warning.
    */
   ephemeral?: boolean;
+  /** The job's `cache`, whose `maxAge` a snapshot holding the name may be past. */
+  cache?: CacheConfig;
 }
 
 /** A snapshot of a job's machine. */
@@ -556,7 +558,7 @@ const createNamedSnapshot = async (
   handle: MachineHandle,
   jobPath: string,
   stepId: string,
-  { target, exclude, broken, ephemeral }: SnapshotCache,
+  { target, exclude, broken, ephemeral, cache }: SnapshotCache,
 ): Promise<TakenSnapshot> => {
   const name = target.name;
 
@@ -591,6 +593,7 @@ const createNamedSnapshot = async (
       run,
       `${stepId}${scopeSeparator}name-taken`,
       name,
+      cache,
       exclude,
       broken,
     );