Allow checking out other repos in CI

.changeset/ci-checkout-repo.md

@@ -0,0 +1,9 @@
+---
+"@inngest/ci": minor
+---
+
+`checkout()` and `files()` can now read another repository or ref, including in runs whose trigger has no repository.
+
+```ts
+await checkout({ repo: "acme/shared-config", ref: "main" });
+```

packages/ci/README.md

@@ -51,6 +51,30 @@ const test = ci.job({ id: "test", from: base }, async () => {
 
 A pipeline run is one trace. `lint` and `test` start from a snapshot of `base`, and a failed command runs again without rerunning the jobs that passed.
 
+## Another repository or ref
+
+`checkout()` and `files()` take `repo` and `ref`. A ref is a branch, tag or commit, and it's resolved to a commit once per run.
+
+```ts
+await checkout({ repo: "acme/platform" });                // default branch
+await checkout({ repo: "acme/platform", ref: "v2.1.0" });
+await checkout({ ref: "main" });                          // this repository
+
+const image = ci.job(
+  {
+    id: "image",
+    cache: {
+      key: files("images/node-base/**", { repo: "acme/platform", ref: "main" }),
+    },
+  },
+  async () => {
+    await checkout({ repo: "acme/platform", ref: "main" });
+  },
+);
+```
+
+A run whose trigger has no repository, like a cron, needs `repo`. The GitHub App must have access to the repository. Locally, `checkout()` still uploads your working tree unless you name another repository or a `ref`.
+
 ## Install
 
 ```bash

packages/ci/src/cache/cache.ts

@@ -10,22 +10,33 @@
  * @module
  */
 
+import { NonRetriableError } from "inngest";
+import { mapGitHubError, rest } from "../github/rest.ts";
+import type { ResolvedSource } from "../github/source.ts";
+import {
+  octokitForSource,
+  resolveSource,
+  targetsRunRepo,
+} from "../github/source.ts";
 import { ciRun } from "../pipeline/metadata.ts";
 import type { StepSpec } from "../pipeline/names.ts";
 import { steps } from "../pipeline/names.ts";
 import type { CiJobScope, CiRunScope } from "../pipeline/scope.ts";
-import { countApi, joinId, rootRunIdOf } from "../pipeline/scope.ts";
+import { countApi, joinId, localRepo, rootRunIdOf } from "../pipeline/scope.ts";
 import type {
   CacheConfig,
   CacheKey,
   CacheKeyPart,
+  FilesOptions,
   JobConfig,
   RepoContext,
 } from "../types.ts";
 import {
+  filterPaths,
   formatRelative,
   hash,
   isSnapshotNotFound,
+  parseRepo,
   stableStringify,
 } from "../util.ts";
 import { formatName } from "./names.ts";
@@ -87,8 +98,9 @@ const sharedScopes = (
  * Resolve the parts of a cache key into a single hash.
  *
  * `files()` parts are resolved against the repository: the git tree for the
- * run's commit on GitHub, or the working tree locally, so a key changes
- * exactly when the matched files do.
+ * run's commit on GitHub (or for the commit its `repo` and `ref` resolved to),
+ * or the working tree locally, so a key changes exactly when the matched files
+ * do.
  */
 export const resolveCacheKey = async (
   run: CiRunScope,
@@ -102,47 +114,115 @@ export const resolveCacheKey = async (
     return hash(await key());
   }
 
-  const parts = Array.isArray(key) ? key : [key];
   const resolved: string[] = [];
 
-  for (const part of parts) {
-    if (typeof part === "string") {
-      resolved.push(part);
+  for (const part of keyParts(key)) {
+    resolved.push(
+      typeof part === "string" ? part : await resolveFilesPart(run, part),
+    );
+  }
 
-      continue;
-    }
+  return hash(resolved.join("\0"));
+};
 
-    resolved.push(await resolveFilesPart(run, part));
+/** The parts of a cache key, which a function key doesn't have. */
+const keyParts = (key: CacheKey | undefined): (CacheKeyPart | string)[] => {
+  if (key === undefined || typeof key === "function") {
+    return [];
   }
 
-  return hash(resolved.join("\0"));
+  return Array.isArray(key) ? key : [key];
+};
+
+/**
+ * Resolve the repository and ref of every `files()` part that reads another
+ * commit than the run's own, each in a memoized step. A key is worked out
+ * inside a step, which can't start another, so this runs just before it.
+ */
+export const resolveKeySources = async (
+  run: CiRunScope,
+  key: CacheKey | undefined,
+): Promise<void> => {
+  for (const part of keyParts(key)) {
+    if (typeof part !== "string" && !targetsRunRepo(run, part)) {
+      await resolveSource(run, part);
+    }
+  }
 };
 
 const resolveFilesPart = async (
   run: CiRunScope,
   part: CacheKeyPart,
 ): Promise<string> => {
-  const repo = run.repo;
+  if (!targetsRunRepo(run, part)) {
+    return hashRemoteFiles(run, await resolveSource(run, part), part);
+  }
+
+  const local = localRepo(run);
 
-  if (repo?.local && process.env.INNGEST_CI_GITHUB !== "live") {
+  if (local) {
     const { hashLocalFiles } = await import("./localCache.ts");
 
-    return hashLocalFiles(repo.local.path, part.patterns);
+    return hashLocalFiles(local.path, part.patterns);
   }
 
+  const repo = run.repo;
+
   if (!repo) {
     return `files:${part.patterns.join(",")}`;
   }
 
-  const { rest } = await import("../github/rest.ts");
-  const { filterPaths } = await import("../util.ts");
-
   const tree = await rest.git.getTree({
     tree_sha: repo.sha,
     recursive: "1",
   });
 
-  const entries = (tree.tree ?? [])
+  return hashTree(tree, part.patterns, repo.fullName);
+};
+
+/**
+ * Hash the matched files of another repository or ref, at the commit it
+ * resolved to, with a client for the installation that can read it.
+ */
+const hashRemoteFiles = async (
+  run: CiRunScope,
+  source: ResolvedSource,
+  part: CacheKeyPart,
+): Promise<string> => {
+  const octokit = await octokitForSource(run, source);
+
+  try {
+    const { data } = await octokit.rest.git.getTree({
+      owner: source.owner,
+      repo: source.name,
+      tree_sha: source.sha,
+      recursive: "1",
+    });
+
+    return hashTree(data, part.patterns, source.fullName);
+  } catch (error) {
+    throw mapGitHubError(error) ?? error;
+  }
+};
+
+/** A hash of the blobs in a git tree that match the patterns. */
+const hashTree = (
+  listing: {
+    tree?: { type?: string; path?: string; sha?: string }[];
+    truncated?: boolean;
+  },
+  patterns: string[],
+  fullName: string,
+): string => {
+  // A truncated listing would hash only some of the files, and a key that
+  // ignores the rest never changes when they do.
+  if (listing.truncated) {
+    throw new NonRetriableError(
+      `\`${fullName}\` is too large to hash \`files()\` from GitHub's tree API, which cut the file listing short. Key the job on a string instead.`,
+    );
+  }
+
+  const entries = (listing.tree ?? [])
     .filter((entry) => {
       return entry.type === "blob" && entry.path;
     })
@@ -154,7 +234,7 @@ const resolveFilesPart = async (
     entries.map((entry) => {
       return entry.path;
     }),
-    { include: part.patterns },
+    { include: patterns },
   );
 
   const byPath = new Map(
@@ -270,6 +350,8 @@ export const cacheTarget = async (
 
   countApi("cache");
 
+  await resolveKeySources(run, cache.key);
+
   const ownKey = await ciRun(run, steps.cacheKey(job.path), () => {
     return jobCacheKey(run, cache, input, base);
   });
@@ -710,14 +792,43 @@ export const describeCached = (
  * Contents are read from the git tree for the run's commit, or from the
  * working tree locally, so uncommitted changes change the key too.
  *
+ * End with `{ repo, ref }` to read another repository or ref. `ref` is a
+ * branch, tag or commit, and `repo` is `"owner/name"` with `ref` defaulting to
+ * its default branch. The ref is resolved to a commit once per run, so a branch
+ * that moves changes the key exactly when the matched files change. Locally,
+ * this reads from GitHub too, unless it names the local repository with no
+ * `ref`.
+ *
+ * ```ts
+ * cache: {
+ *   key: files("images/node-base/**", { repo: "acme/platform", ref: "main" }),
+ * }
+ * ```
+ *
  * Patterns support `**`, `*`, `?`, and `{a,b}`.
  */
 export const files = (
-  /** Glob patterns for the files to hash. */ ...patterns: string[]
+  /** Glob patterns for the files to hash, then optionally where to read them from. */
+  ...args: string[] | [...patterns: string[], options: FilesOptions]
 ): CacheKeyPart => {
+  const list: (string | FilesOptions)[] = args;
+  const last = list.at(-1);
+  const options = typeof last === "object" ? last : undefined;
+
+  // Checked here, where the pipeline is defined, so a bad `repo` fails at boot.
+  if (options?.repo !== undefined) {
+    parseRepo(options.repo);
+  }
+
+  const patterns = list.filter((arg): arg is string => {
+    return typeof arg === "string";
+  });
+
   return {
     kind: "inngest/ci.cacheKeyPart",
     type: "files",
     patterns,
+    ...(options?.repo === undefined ? {} : { repo: options.repo }),
+    ...(options?.ref === undefined ? {} : { ref: options.ref }),
   };
 };