diff --git a/.env.example b/.env.example index 3fffde6d1..20a1c2c1f 100644 --- a/.env.example +++ b/.env.example @@ -190,6 +190,26 @@ W3DS_EXTRA_RESERVED_USERNAMES="" # Minimum eID Wallet version accepted. Temporary - drops out after the rollout. W3DS_MIN_WALLET_VERSION="0.4.0" +# Forgejo code sync (services/forgejo-code-sync) +# Syncs commits pushed to GitW3 into the pushing author's eVault, resolved via +# the login_name the bridge above writes on sign-in. See +# docs/superpowers/specs/2026-08-14-forgejo-code-sync-design.md. +FORGEJO_SYNC_PUBLIC_URL="http://localhost:4300" +FORGEJO_SYNC_PORT=4300 +# HMAC secret configured on the Forgejo system webhook (POST /api/v1/admin/hooks). +# Compared against the raw, unprefixed X-Forgejo-Signature header. +FORGEJO_WEBHOOK_SECRET="replace-with-a-strong-secret" +# GitW3's base URL, for the admin Users API call and diff fetching. +FORGEJO_API_URL="http://localhost:3080" +# PAT on a dedicated site-admin service account created for this service alone +# (never a shared human admin's token) - scopes read:user,read:repository. +# read:user alone is not enough: GitW3 only returns login_name to a caller whose +# account has IsAdmin=true, regardless of token scope. See the spec's Trust model. +FORGEJO_ADMIN_TOKEN="" +# Commit diffs are uploaded to the DO_SPACES_* bucket configured above (the +# same one evault-core's StorageService.ts uses) rather than inlined into the +# eVault write - no service-specific S3 keys needed here, just those. + # --- Deploying GitW3 and the bridge together ------------------------------- # Only needed for docker-compose.gitw3.yml. Local development uses the block # above and runs the bridge with `pnpm --filter w3ds-oidc-bridge dev`. diff --git a/docker/Dockerfile.forgejo-code-sync b/docker/Dockerfile.forgejo-code-sync new file mode 100644 index 000000000..7492c4bfa --- /dev/null +++ b/docker/Dockerfile.forgejo-code-sync @@ -0,0 +1,42 @@ +FROM node:20-alpine AS base +RUN apk add --no-cache libc6-compat python3 make g++ +WORKDIR /app + +ENV CI=true +ENV PYTHON=/usr/bin/python3 +RUN ln -sf python3 /usr/bin/python + +# --- +FROM base AS prepare +RUN npm install -g pnpm@10.25.0 turbo@^2 +COPY . . +RUN turbo prune forgejo-code-sync --docker + +# --- +FROM base AS builder +RUN npm install -g pnpm@10.25.0 +# Dependencies first, since they change far less often than the source. +COPY --from=prepare /app/out/json/ . +RUN pnpm install --frozen-lockfile +COPY --from=prepare /app/out/full/ . +RUN pnpm turbo build --filter=forgejo-code-sync + +# --- +FROM base AS runner +COPY --from=builder /app/package.json ./ +COPY --from=builder /app/pnpm-workspace.yaml ./ +COPY --from=builder /app/pnpm-lock.yaml ./ + +COPY --from=builder /app/services/forgejo-code-sync/dist ./services/forgejo-code-sync/dist +COPY --from=builder /app/services/forgejo-code-sync/package.json ./services/forgejo-code-sync/ +COPY --from=builder /app/services/forgejo-code-sync/node_modules ./services/forgejo-code-sync/node_modules +COPY --from=builder /app/node_modules ./node_modules + +WORKDIR /app/services/forgejo-code-sync + +# Keep in step with FORGEJO_SYNC_PORT. +EXPOSE 4300 +HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ + CMD node -e "require('http').get('http://localhost:4300/healthz', (r) => {process.exit(r.statusCode === 200 ? 0 : 1)}).on('error', () => process.exit(1))" + +CMD ["node", "dist/index.js"] diff --git a/docs/superpowers/plans/2026-08-14-forgejo-code-sync-plan.md b/docs/superpowers/plans/2026-08-14-forgejo-code-sync-plan.md new file mode 100644 index 000000000..8c597fe43 --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-forgejo-code-sync-plan.md @@ -0,0 +1,349 @@ +# Implementation plan — Forgejo code sync + +**Spec:** [2026-08-14-forgejo-code-sync-design.md](../specs/2026-08-14-forgejo-code-sync-design.md) + +**Status: Phases 0–5 built and passing (89 tests), Phase 6 run once against a live GitW3 instance.** Tasks 1.5, 4.1 +and 4.2 below describe the *original* cap-then-inline diff design, superseded after that live run found it wrong in +two ways — see the spec's "What gets written" for the corrected design (diff always uploaded to S3, never inlined; +fetched from the API router's `git/commits/{sha}.diff`, not the web router's `commit/{sha}.diff`, which doesn't +authenticate a PAT for a private repo at all). Left as-written below for the historical record of what was actually +tried and why it changed, rather than silently edited to look right in hindsight. + +Rationale lives in the spec; this document is the order of work and how each step is proved. Every task states its +verification. A task is not done until its verification passes. + +Conventions taken from the bridge's own plan: `vitest run` as the `test` script, Biome for format and lint, `tsc +--noEmit` as `check-types`, config read from the root `.env` through a `required()` helper that throws at startup. + +Phases 1–3 are fully testable with no live GitW3 instance, no eVault, and no admin token — everything in them is a +pure function or stubbed at its one external call. Do not skip ahead: the identity-resolution and ACL logic is where +the spec's traps live (pusher vs. commit author, `login_name`'s `@` check, `Repo.Private` → ACL), and it is the part +that can be tested exhaustively before anything depends on a real Forgejo instance being reachable. + +--- + +## Phase 0 — Scaffolding + +**0.1 Create the package.** `services/forgejo-code-sync/` with `package.json` (name `forgejo-code-sync`, `type: +module`), `tsconfig.json`, `tsconfig.build.json`, `README.md`. Scripts: `dev`, `build`, `start`, `test`, `test:watch`, +`check`, `check-types` — same set as `services/w3ds-oidc-bridge/package.json`. Dependencies: `express`, `dotenv`, +`graphql-request` (matches `EVaultService.ts`'s client, not a new GraphQL dependency). Dev: `vitest`, `typescript`, +`@types/*`, `tsx`. + +Picked up by `services/*` in `pnpm-workspace.yaml` — no workspace change needed. + +> **Verify:** `pnpm install` resolves; `pnpm --filter forgejo-code-sync check-types` passes on an empty `src/index.ts`. + +**0.2 Add env keys to `.env.example`** — the full table from the spec's [Deployment](../specs/2026-08-14-forgejo-code-sync-design.md#deployment) +section. Do not touch `.env`. + +> **Verify:** every key in the spec's table appears in `.env.example`. + +**0.3 Mint the ontology schema.** `services/ontology/schemas/codeCommit.json`, generating a fresh `schemaId` (a UUID +— check it doesn't collide with an existing one in the directory), fields as specced under +[What gets written](../specs/2026-08-14-forgejo-code-sync-design.md#what-gets-written). The ontology service +(`services/ontology/src/index.js`) loads every `.json` file in that directory at startup with no registration step +beyond adding the file. + +> **Verify:** `pnpm --filter ontology dev` (or however that service is started locally) logs one more loaded schema +> than before; `GET /schemas/:uuid` on the new id returns the file's contents unchanged. + +*Commit: `chore(forgejo-code-sync): scaffold the service package and mint the CodeCommit ontology schema`* + +--- + +## Phase 1 — The pure core + +No HTTP, no network calls in this phase. Everything here is a function, mirroring the bridge's own Phase 1 split. + +**1.1 `src/config.ts`.** Parse and validate env with `required()`, mirroring +[awareness-service's config.ts](../../../services/awareness-service/api/src/config.ts) and the bridge's own. Load the +root `.env` by relative path. + +> **Verify:** unit tests — missing key throws naming that key. + +**1.2 `src/identity.ts`, the pure half.** A function `enameFromLoginName(loginName: string): string | null` — returns +the ename when `loginName` starts with `@`, `null` otherwise. This is the entire "no linked eVault" detection logic +from the spec's [Identity resolution](../specs/2026-08-14-forgejo-code-sync-design.md#identity-resolution-pusher--eName) +section, kept as a one-line pure function precisely so it can be tested without the admin API call that surrounds it +(added in Phase 3). + +> **Verify:** unit tests — `"@alice"` → `"@alice"`; `"alice"` (no `@`, an ordinary password account) → `null`; empty +> string → `null`. + +**1.3 `src/webhook/signature.ts`.** `verifyForgejoSignature(rawBody: Buffer, secret: string, header: string | +undefined): boolean`, exactly the implementation already written out in the spec's +[Forgejo webhook side](../specs/2026-08-14-forgejo-code-sync-design.md#forgejo-webhook-side) section: `createHmac` +over the raw bytes, compared with `timingSafeEqual` against the **unprefixed** hex digest, length-checked first. + +> **Verify:** unit tests — a valid signature over a fixed body/secret pair accepted; one byte flipped in the body +> rejected; a `sha256=`-prefixed value (the GitHub-boilerplate trap the spec calls out) rejected as a regression +> guard, not just an absent header; a missing header rejected without throwing. + +**1.4 `src/evault/acl.ts`.** `deriveAcl(repoIsPrivate: boolean, eName: string): string[]` — `["*"]` when `false`, +`[eName]` when `true`. + +**Confirmed against `infrastructure/evault-core` source, not guessed.** `typedefs.ts:153,283,292` — `acl: [String!]!`, +a plain string array. `"*"` is special-cased as public everywhere it's checked +(`vault-access-guard.ts`'s `checkAccess`/`filterEnvelopesByAccess`). Every write in this entire codebase uses +`acl: ["*"]` except one: `infrastructure/evault-core/src/services/BindingDocumentService.ts:298,378` writes +`acl: [normalizedSubject]` / `acl: [bindingDocument.subject]` — a single-entry array holding the subject's eName. +That's the one real precedent for a restricted ACL anywhere in the codebase, and it's what `[eName]` is modelled on. + +**Known limitation of that protection, confirmed while checking the syntax — document it, don't try to fix it +here.** `vault-access-guard.ts`'s `checkAccess` (the single-envelope-by-ID lookup path) grants `hasAccess: true` +whenever the caller presents **any valid Registry-issued Bearer token from any certified platform** — not +specifically the platform that wrote the envelope — without consulting `metaEnvelope.acl` in that branch at all. The +ACL is only actually enforced when no valid Bearer token is present (an anonymous request), and in +`filterEnvelopesByAccess` (the bulk `metaEnvelopes` list query, which has no such bypass). So `[eName]` reliably +blocks anonymous reads and keeps the envelope out of another platform's list-query results, but does **not** block a +different certified platform from reading the exact same envelope via a direct by-ID lookup if it already knows the +envelope's ID and the right `X-ENAME`. This is `infrastructure/evault-core`'s existing authorization model, not a bug +introduced here, and changing it is out of scope for this service — but "owner-only" in this codebase means "not +public or anonymously/cross-platform-listable," not "cryptographically restricted to the owner." Say so in the code +comment above `deriveAcl`, not just in this plan, so nobody reads the function name later and assumes more than it +delivers. + +> **Verify:** unit tests — `(true, "@alice")` → `["@alice"]`; `(false, "@alice")` → `["*"]`. The doc comment above the +> function states both the `BindingDocumentService.ts` precedent and the by-ID enforcement gap, not just the return +> shape. + +**1.5 `src/content/diffSize.ts`.** `shouldInline(diffBytes: number, maxBytes: number): boolean` — the cap decision +from [What gets written](../specs/2026-08-14-forgejo-code-sync-design.md#what-gets-written), isolated so the +size-threshold logic doesn't get buried inside the HTTP-fetching code written in Phase 4. + +> **Verify:** unit tests — at, above, and below the boundary. + +*Commit: `feat(forgejo-code-sync): config, identity, signature verification, ACL derivation and diff-size pure core`* + +--- + +## Phase 2 — Webhook receipt and the queue + +**2.1 `src/queue.ts`.** A persisted queue for commit-sync tasks, per the spec's +[Delivery reliability](../specs/2026-08-14-forgejo-code-sync-design.md#delivery-reliability-no-safety-net-from-forgejo) +section — Forgejo will never redeliver a failed webhook, so this service's own retry is the only safety net. Minimum +shape: `enqueue(task)`, `markSucceeded(id)`, `markFailed(id, error)` with backoff scheduling, and a status distinct +for "pending," "retrying," and "exhausted, needs attention" — the last must be distinguishable from an ordinary +"skipped, no linked eVault" outcome in whatever this emits to logs/metrics, per the spec's explicit requirement that +those two must never look the same from outside. + +Backing store is an open implementation choice within this phase — SQLite file, a table in whatever Postgres this +service ends up with, even a durable on-disk JSON queue for a first cut — but it must survive a process restart; an +in-memory-only queue does not satisfy the spec's requirement and should not be treated as a placeholder that's "good +enough for now," since the whole point of this phase is that Forgejo gives this service no second chance. + +> **Verify:** unit tests — a task that fails is retried with backoff, not dropped; one that exhausts its retry budget +> is marked exhausted, not silently removed; the queue's contents survive a simulated restart (reload from the +> backing store and confirm the pending task is still there). + +**2.2 `src/webhook/push.ts`.** `POST /webhook` — capture the raw request body (`express.json({ verify: (req, res, +buf) => { req.rawBody = buf } })`, per the spec's explicit trap about re-serialized bodies breaking signature +verification), check it with `verifyForgejoSignature` from 1.3, parse the `PushPayload`, and for each commit call +`queue.enqueue(...)` with everything downstream processing will need: `pusher.username`, `repo` (owner/name), +`repo.private`, `ref`, the commit's `id`/`message`/`timestamp`/`added`/`removed`/`modified`, and the commit's own URL +(for the `diffUrl` fallback). Responds `200` once every commit in the delivery is durably queued — not once they're +processed, per the spec's architecture diagram. + +> **Verify:** unit tests — a request with a valid signature and N commits enqueues N tasks and returns 200; an +> invalid signature is rejected before anything is queued; a request with 0 commits (e.g. a tag push, if the system +> webhook fires on non-branch refs too) queues nothing and still returns 200 rather than erroring. + +*Commit: `feat(forgejo-code-sync): webhook receipt, raw-body signature verification and the persisted retry queue`* + +--- + +## Phase 3 — Identity resolution and the eVault write + +**3.1 `src/identity.ts`, the network half.** `resolveEname(username: string): Promise` — `GET +{FORGEJO_API_URL}/api/v1/users/{username}` with `FORGEJO_ADMIN_TOKEN`, read `login_name` off the response, pass it +through `enameFromLoginName` from 1.2. Cache successful and null-mapping results with a long TTL; a 404 (account +deleted) evicts the cache entry rather than being retried on the usual backoff schedule, since a deleted account isn't +a transient failure. + +> **Verify:** unit tests with the HTTP call stubbed — a `login_name` starting with `@` resolves to that ename; one +> that doesn't returns `null`; a second call for the same username within the TTL doesn't re-hit the stub; a 404 +> evicts a previously-cached entry. + +**3.2 `src/evault/client.ts`.** The certify-then-per-eName-GraphQL-client pattern, copied from +[`EVaultService.ts`](../../../platforms/calendar/api/src/services/EVaultService.ts)'s shape: `ensurePlatformToken()` +caches the Registry certification until near expiry, `getClient(eName)` returns a `GraphQLClient` with `Authorization` +and `X-ENAME` headers, `writeCommit(eName, payload, acl)` calls `createMetaEnvelope` with the `codeCommit` ontology id +minted in 0.3. + +> **Verify:** unit tests with `fetch`/`GraphQLClient` stubbed — certification is requested once and reused across +> multiple `writeCommit` calls within the cache window; a new token is requested after simulated expiry; the mutation +> variables sent match `{ ontology: , payload: {...}, acl }` exactly. + +**3.3 Wire the queue's drain loop.** The consumer side of `queue.ts`: for each dequeued task, resolve the eName +(3.1) — `null` marks the task done-and-skipped, not failed, and must not enter the retry path — derive the ACL from +the task's `repo.private` (1.4), fetch or cap the diff (Phase 4), and call `writeCommit` (3.2). A thrown error at any +step marks the task failed and lets the queue's backoff handle the retry. + +> **Verify:** integration test (all externals stubbed) — a task for a `login_name`-having pusher on a public repo +> ends in a `createMetaEnvelope` call with `acl: ["*"]`; one on a private repo ends with the owner-only ACL; a task +> for an unlinked account is marked done without ever calling `writeCommit`; a stubbed eVault failure marks the task +> failed and leaves it in the queue for retry, not silently dropped. + +*Commit: `feat(forgejo-code-sync): identity resolution, eVault client, and the queue's drain loop`* + +--- + +## Phase 4 — Diff fetching + +**Superseded — see the status note at the top of this document.** What actually shipped: `fetchDiff(task, eName): +Promise`, fetching from `GET {FORGEJO_API_URL}/api/v1/repos/{repo}/git/commits/{sha}.diff` (the API router, +not the web router below — confirmed live that the web router never authenticates a PAT for a private repo), then +uploading the result to S3 via `storage/s3.ts` and returning that URL. No size cap; throws on any failure instead of +degrading to a fallback, since there's no longer a lesser fallback to degrade to. + +**4.1 `src/content/diff.ts`, as originally planned.** `fetchDiff(repo, sha, token): Promise<{ diff: string } | { diffUrl: string }>` — `GET +{FORGEJO_API_URL}/{owner}/{repo}/commit/{sha}.diff` with `FORGEJO_ADMIN_TOKEN` (needs `read:repository` per the +spec's [Trust model](../specs/2026-08-14-forgejo-code-sync-design.md#trust-model)); if the response exceeds +`FORGEJO_SYNC_DIFF_MAX_BYTES` (checked via `Content-Length` where present, or a streamed byte count where it isn't) +or the request fails outright, return the `diffUrl` fallback using the commit's own GitW3 URL or `Repo.CompareURL` +from the original webhook payload — no diff field attempted in that case, per the spec. + +> **Verify:** unit tests with the HTTP call stubbed — a small diff is returned inline; one over the cap returns +> `diffUrl` with no `diff` field; a stubbed network failure degrades to `diffUrl` rather than throwing out of the +> queue task (the task should still succeed and write an envelope with `diffUrl` set, not fail and retry forever on +> an oversized or permanently-erroring diff). + +**4.2 Wire into the drain loop from 3.3.** + +> **Verify:** the Phase 3.3 integration test extended with a real (stubbed) diff fetch — the written envelope's +> `diff`/`diffUrl` fields match the cap decision. + +*Commit: `feat(forgejo-code-sync): commit diff fetching with size cap and URL fallback`* + +--- + +## Phase 5 — Wiring and packaging + +**5.1 `src/index.ts`.** Wire the webhook route, start the queue's drain loop, add `/healthz`, log the resolved +`FORGEJO_API_URL` and queue backend at startup, fail fast on a config error. + +> **Verify:** `pnpm --filter forgejo-code-sync dev` starts; `curl /healthz` returns 200; starting with a missing env +> key exits non-zero naming the key. + +**5.2 `docker/Dockerfile.forgejo-code-sync`**, following the `docker/Dockerfile.` convention used by the +bridge's own Dockerfile. + +**5.3 `services/forgejo-code-sync/README.md`** — what it is, the two contracts in one paragraph each (mirroring the +bridge's README structure), the env table, how to run it locally, and how to provision the site-admin service account +and register the system webhook (5.4 below). Link the spec rather than restating it. + +**5.4 Provisioning script.** A small script (shell or `ts-node`, whichever matches how 5.4 ends up being run) that +calls `POST /api/v1/admin/hooks` on `FORGEJO_API_URL` with `FORGEJO_ADMIN_TOKEN` to register the system webhook +pointed at this service's `/webhook` endpoint, per the spec's [Deployment](../specs/2026-08-14-forgejo-code-sync-design.md#deployment) +section — the `POST /api/v1/admin/hooks` route confirmed against GitW3's own source, not by analogy. Idempotent if +practical (check for an existing hook with this service's URL before creating a second — `GET /api/v1/admin/hooks` is +paginated, so the check must page through all results, not just the first page's default limit), matching the spirit +of `docker/gitw3-register-auth-source.sh` even though the mechanism (REST API, not CLI) differs. + +**GitW3-verified request shape, and a trap in it**: checked directly against +`routers/api/v1/admin/hooks.go`/`routers/api/v1/utils/hook.go` (`addHook`, `checkCreateHookOption`) — + +```jsonc +// POST /api/v1/admin/hooks +{ + "type": "forgejo", // required; "gitea" also works — anything else (e.g. "slack") changes the + // payload shape and drops the signature headers this service depends on + "config": { + "url": "", // required, validated as a URL + "content_type": "json", // required; only "json" or "form" are valid — "form" would break + // the raw-body JSON signing this service's verification assumes + "secret": "", // NOT required by the API — omitting it is accepted with 201 + "is_system_webhook": "true" // undocumented in CreateHookOption's own shape — see the trap below + }, + "events": ["push"], // optional — omitted entirely defaults to exactly ["push"] server-side, but + // pass it explicitly rather than relying on that default + "active": true // required to actually fire. Defaults to false if omitted. +} +``` + +**The trap**: `active` defaults to `false` (`CreateHookOption.Active bool`, zero value). A request that omits it +still returns `201 Created` with a real hook ID — Site Administration → Webhooks shows it, `GET /admin/hooks` lists +it — but `IsActive: form.Active` means it silently never delivers a single push. This is a false-positive success at +exactly the boundary the [Deployment](../specs/2026-08-14-forgejo-code-sync-design.md#deployment) verification step +below checks ("the hook appears in Site Administration") — "appears" is not "active," and the current verification +wording doesn't distinguish them. A second, lower-severity version of the same class of mistake: `config.secret` is +accepted as absent (`checkCreateHookOption` only requires `url` and `content_type`) — an omitted secret means +Forgejo signs with an empty key, so `X-Forgejo-Signature` arrives empty and `verifyForgejoSignature` correctly +rejects every delivery, but from the outside this looks identical to "webhook never configured," not "webhook +misconfigured," making it slower to diagnose than the `active` trap. + +**A third trap, found only by testing against a live instance — this JSON body's own comment above understated it.** +`POST /admin/hooks` creates a "default" webhook, not a "system" one, unless `config.is_system_webhook` is the +*string* `"true"` — read out of `CreateHookOption`'s free-form `config` map by +`routers/api/v1/utils/hook.go`'s `addHook`, not a documented top-level field. Reproduced directly: without it, the +create call still returns `201`, but the resulting hook (a) is invisible to `GET /admin/hooks` — that endpoint's +`GetSystemWebhooks` filters `is_system_webhook=true` at the DB layer — and (b) is only copied into repos created +*after* it's added, never applying retroactively to an existing repo. Both failure modes are silent successes, +exactly like the `active` trap, and compound with it: a hook that's both inactive and non-system passes every +naive check (`201` returned, script exits `0`) while doing nothing at all, for two independent reasons. Deleting a +stray non-system hook created this way has no API or CLI path either — `GetDefaultWebhooks` (the model function that +would list it) is only called from the web admin UI's own handler, `routers/web/admin/hooks.go`, never exposed over +`/api/v1/`. + +> **Verify:** run against a local GitW3 instance with a real site-admin token — the hook appears in Site +> Administration → Webhooks **and its Active toggle is on** (not just present in the list); a manual "Test Delivery" +> from that same screen (or an actual push) results in a real `POST` hitting this service's `/webhook`, not just a +> row existing in the hooks table; running the script twice does not create a duplicate. + +*Commit: `chore(forgejo-code-sync): dockerfile, service README, and system-webhook provisioning script`* + +--- + +## Phase 6 — Local end-to-end + +No new code in this phase — it's where the acceptance criteria are actually exercised, mirroring the bridge's own +Phase 5. + +**6.1 Provision the site-admin service account** on a local GitW3 instance, generate its PAT with `read:user, +read:repository` scopes, run the 5.4 script to register the webhook. + +**6.2 Link a real W3DS identity** to a GitW3 account via the bridge's own local flow (Dev Sandbox, per the bridge's +README), so a real `login_name` exists to resolve against. + +**6.3 Push a commit** from that linked account to a public repo, then a private one. + +> **Verify:** the public-repo push produces an envelope with `acl: ["*"]`; the private-repo push produces one with +> the owner-only ACL; both carry the correct `authorEName`, matching the eName the account was linked with; a diff +> under the configured cap is inlined, and lowering `FORGEJO_SYNC_DIFF_MAX_BYTES` to something tiny and repeating +> produces `diffUrl` instead. + +**6.4 Push from an unlinked account** (a plain password-registered GitW3 user). + +> **Verify:** no envelope is written; the queue shows the task as done-and-skipped, not failed; nothing alerts. + +**6.5 Simulate a dropped delivery** — stop the eVault (or point `PUBLIC_EVAULT_SERVER_URI` at something unreachable) +before a push, then restore it. + +> **Verify:** the task is retried and eventually succeeds once the eVault is reachable again, with no restart of this +> service required; if the service itself is restarted mid-retry, the task is still there afterward (queue +> persistence from 2.1). + +--- + +## Order dependencies + +``` +0 ──▶ 1 ──▶ 2 ──▶ 3 ──▶ 4 ──▶ 5 ──▶ 6 + ▲ + deployment path (open, see spec) — + does not block anything above; + only blocks an actual staging rollout +``` + +Phase 1 has no dependency on a live GitW3 instance, an eVault, or an admin token — everything in it is stubbed or +pure, same discipline as the bridge's own plan. Phase 2's queue exists before Phase 3's network calls specifically so +that Phase 3 can be built and tested against a queue that already has the right failure-handling contract, rather than +retrofitting persistence onto code that was written assuming every call succeeds. + +## Out of scope + +Re-scanning already-synced envelopes when a repo's visibility changes after the fact (the spec's named, accepted ACL +limitation). A UI or query surface for a person to browse their own synced commits — this plan only covers the write +path. Handling merge commits or squash-merges any differently from an ordinary commit — the webhook payload doesn't +distinguish them, and neither does this design. diff --git a/docs/superpowers/specs/2026-08-14-forgejo-code-sync-design.md b/docs/superpowers/specs/2026-08-14-forgejo-code-sync-design.md new file mode 100644 index 000000000..7109b5885 --- /dev/null +++ b/docs/superpowers/specs/2026-08-14-forgejo-code-sync-design.md @@ -0,0 +1,810 @@ +# Forgejo code sync — commits into the author's eVault + +**Date:** 2026-08-14 +**Branch:** `feat/forgejoCodeSyncWithEvault` (based off `feat/w3ds-oidc-bridge`) +**Status:** reviewed and ready to build. ACL, admin-token custody, and delivery-reliability decisions made 2026-08-14; +`login_name` carrying the full eName confirmed end to end by source chain across bridge and GitW3 (see +[Verification status](#verification-status)). Deployment path remains open, inherited from the bridge's own +unresolved blocker — not re-litigated here, and does not block implementation. + +Paths given without a link refer to upstream Gitea/Forgejo source, not this repository — see +[Verification status](#verification-status) for how confidently each claim is held. + +## Problem + +[w3ds-oidc-bridge](../../../services/w3ds-oidc-bridge/README.md) lets a person sign into GitW3 with their W3DS +identity. Once they can sign in, the natural next step is that the code they push becomes part of their own record — +written into their eVault rather than living only on GitW3's server. + +Same governing constraint as the bridge: GitW3 is kept at a patch surface of zero, so nothing here can touch Forgejo's +source. It has to work through a surface Forgejo already exposes — outbound webhooks and its REST API — the same way +the bridge worked through OIDC rather than a plugin API it doesn't have. + +## Approach + +A new service, `services/forgejo-code-sync`, registered as a **system webhook** (fires for every repository, no +per-repo setup — matches the "sync every push, best-effort" decision below) receiving Forgejo's `push` event. For each +commit, it resolves the pusher to an eName, fetches the diff, and writes it as a MetaEnvelope into that person's +eVault. + +Three decisions were made before this draft, each with a real alternative that was set aside: + +**Identity resolution reuses the OIDC bridge's link**, rather than a new mapping table this service owns itself. The +bridge already causes GitW3 to record the ename as `user.login_name` on first sign-in — see +[Identity resolution](#identity-resolution-pusher--eName) — so a second store would just be a second source of truth +for the same fact, and one that could drift from the first if someone re-links. + +**The eVault record is the commit/diff, not a whole-file snapshot.** A snapshot-per-push model was considered and set +aside: it would require this service to hold a full second copy of every synced file's current state and reconcile it +on every push, which is exactly what git itself already does. A commit is the unit GitW3 already emits and the unit a +person would recognise as "what I wrote." + +**Every push is attempted, not opt-in per repository.** A person who has never linked a W3DS identity simply has no +eName to resolve to, and the push is skipped — silently, not as an error, since "no eVault" is the ordinary case for +most GitW3 accounts, not a failure. This is simpler than a per-repo toggle and costs nothing extra per skipped push +beyond the one lookup below. + +## The two contracts + +Same shape as the bridge's document: two protocols this service does not get to choose, then its own design. + +### Forgejo webhook side + +A **system webhook** — Forgejo/Gitea's instance-wide kind, distinct from a *default* webhook (which is only copied +into repos created after it's added). System webhooks fire for every push, on every repo, retroactively, which is what +"best-effort on all pushes" requires. Configured once, by hand or scripted, not per-repo. +([Forgejo webhook docs](https://forgejo.org/docs/latest/user/webhooks/)) + +Payload shape, from Gitea's `modules/structs/hook.go` (Forgejo has not been checked directly — see +[Verification status](#verification-status)): + +```go +type PushPayload struct { + Ref, Before, After, CompareURL string + Commits []*PayloadCommit + HeadCommit *PayloadCommit + Repo *Repository // carries Private bool — see Trust model + Pusher *User // the authenticated Forgejo account that ran git push + Sender *User +} + +type PayloadCommit struct { + ID, Message, URL string + Author, Committer *PayloadUser // free-text git config: name, email, username + Timestamp time.Time + Added, Removed, Modified []string +} +``` + +**`pusher` and each commit's `author` are different things, and only one of them is trustworthy.** `pusher` is the +Forgejo account that authenticated and ran the push. Each commit's `author`/`committer` is whatever `git config +user.name`/`user.email` said on the machine that made the commit — never validated against any Forgejo account, and +trivially set to anyone's name. A rebase, a `git commit --author`, or a laptop with someone else's git config all +produce a mismatch. **This service resolves identity from `pusher`, once per webhook delivery, never from a commit's +own `author` field.** This is the load-bearing trap in this design, in the same category as the bridge's `@` trap — +easy to get right by accident on the happy path, and silently wrong the first time someone force-pushes a rebased +branch authored partly by someone else. + +Signature: `X-Forgejo-Signature` (Forgejo renames some Gitea webhook headers, confirmed for this one; `X-Gitea-Signature` +kept for compatibility), HMAC-SHA256 over the raw request body, hex-encoded, checked against the webhook's configured +secret with a constant-time comparison — same requirement as the bridge's `client_secret` check. + +**GitW3-verified trap**: checked directly against GitW3's own source +(`services/webhook/shared/payloader.go`'s `AddDefaultHeaders`) — `X-Forgejo-Signature` is the **raw hex digest, no +algorithm prefix**. Forgejo sends a GitHub-compatible `X-Hub-Signature-256` header alongside it (same digest, prefixed +`sha256=`), and it's easy to adapt GitHub-webhook-verification boilerplate that strips a `sha256=` prefix before +comparing — pointed at `X-Forgejo-Signature`'s unprefixed value, that strip silently breaks every signature check. +Compare `X-Forgejo-Signature` directly against the hex digest, no prefix handling. + +**The fix, concretely** — two more things beyond the prefix that break this the same way (wrong bytes hashed, or a +timing side-channel), so all three belong in the same implementation, not just the prefix one: + +```ts +import { createHmac, timingSafeEqual } from "node:crypto"; + +function verifyForgejoSignature(rawBody: Buffer, secret: string, header: string | undefined): boolean { + if (!header) return false; + + const expected = createHmac("sha256", secret).update(rawBody).digest("hex"); + const received = Buffer.from(header, "hex"); // X-Forgejo-Signature, used as-is — no "sha256=".slice(7) + const expectedBuf = Buffer.from(expected, "hex"); + + return received.length === expectedBuf.length && timingSafeEqual(received, expectedBuf); +} +``` + +1. **No prefix stripping** — the trap above. +2. **Hash the raw request bytes, not a re-serialized `req.body`.** Forgejo signs the exact bytes it sent on the + wire; if Express's JSON body parser re-`JSON.stringify`s the parsed payload before hashing, key ordering or + whitespace differences make even a correctly-unprefixed comparison fail. Capture the raw buffer explicitly — + `express.json({ verify: (req, res, buf) => { req.rawBody = buf } })` — and hash `req.rawBody`, never `req.body`. +3. **`timingSafeEqual`, not `===`**, same requirement the bridge's `client_secret` check already has — and check + `.length` first, since `timingSafeEqual` throws (rather than returning `false`) on a length mismatch, which an + attacker could otherwise use to distinguish "wrong length" from "wrong bytes." + +### eVault side + +Fixed by `infrastructure/evault-core`'s GraphQL API and the Registry's platform-certification flow, already used by +[`platforms/calendar/api/src/services/EVaultService.ts`](../../../platforms/calendar/api/src/services/EVaultService.ts): +certify once (`POST {registry}/platforms/certification` with this service's own base URL → bearer token, cached until +near expiry), then `createMetaEnvelope(input: { ontology, payload, acl })` over `{PUBLIC_EVAULT_SERVER_URI}/graphql` +with `X-ENAME: ` on each call. The token authenticates the *platform*; the header selects *whose* +eVault the write lands in. This is the pattern to copy — not `PlatformEVaultService.ts` (used by file-manager, +esigner, ecurrency, dreamsync, cerberus), which provisions one eVault owned by the platform itself and is for a +different purpose (platform presence in the Registry, not per-user storage). + +**Unlike the calendar platform, `acl` is not a constant `["*"]` here — see [Trust model](#trust-model).** + +## Identity resolution: pusher → eName + +This is the part with no existing template — every other platform gets the eName from its own login flow. Here, the +person "logging in" (via the bridge, to GitW3) and the event that needs the eName (a push, hours or months later) are +different services, different requests, with nothing linking them but Forgejo's own account record. + +**The webhook payload does not carry it.** Traced to the call site: `services/webhook/notifier.go`'s `PushCommits` +builds `pusher` via `convert.ToUser(ctx, pusher, nil)` — `doer` is `nil`. `services/convert/user.go`'s `ToUser` only +populates `LoginName` (and `SourceID`) when `doer.ID == user.ID || doer.IsAdmin`. With `doer` nil, that's always false. +**The ename is never in the webhook**, regardless of how the pusher authenticated. This was verified against source, +not assumed. + +**It is reachable one call away.** The same `ToUser` function *does* populate `LoginName` for an admin-authenticated +caller. `GET /api/v1/users/{username}` with an admin personal access token returns `login_name` — which, per the +bridge's own design doc, is where GitW3 stores the full ename (`@` included) after W3DS auto-provisioning. So the +flow per push is: read `pusher.username` from the webhook, call the admin Users API once per unique username, read +`login_name` off the response. + +**Detecting "this account has no linked eVault."** Not every GitW3 account signed in through the bridge. A +password-registered account's `login_name` is not an ename. The bridge's claims design guarantees enames always begin +with `@` (`claims.ts`'s sanitiser strips the leading `@` on the way *in*, but `login_name` is what the OIDC flow wrote +verbatim, unstripped — the bridge's own doc confirms `login_name` holds the full ename, `@` included). Treating a +`login_name` that does not start with `@` as "no linked eVault, skip" is a direct, cheap check with no separate +mapping table needed. **Confirmed by source chain, not just the bridge's doc** — see +[Verification status](#verification-status): the bridge's `claims.ts` sets `sub` to the ename verbatim, `@` included, +goth maps `sub` straight to `UserID`, and GitW3 writes `LoginName = gothUser.UserID` verbatim. No stripping happens +anywhere on that path. + +**Cache the mapping.** `login_name` does not change on its own, so resolve once per Forgejo username and cache with a +long TTL, invalidated on a 404 (account deleted) rather than polled. Without this, every push does an extra +admin-authenticated API round trip before any actual sync work starts. + +**The admin token is new trust this design adds — see [Trust model](#trust-model) for its actual required shape**, +which turned out larger than a simple read-only lookup token once checked against GitW3's real permission model. + +## What gets written + +One MetaEnvelope per commit, into the pusher's eVault, on a new ontology schema (`services/ontology/schemas/` has no +existing schema shaped for this — `file.json` is for arbitrary S3-backed blobs, no repo/commit/ref fields). Modelled +on the recent ontology schemas' convention (see `communityActivity.json`, `calendarAvailability.json`): rich +`description` fields, an `authorEName` field for consistency even though the envelope already lives in that person's +own eVault, `additionalProperties` decided deliberately rather than left implicit. + +```jsonc +// services/ontology/schemas/codeCommit.json +{ + "schemaId": "af7b8ea0-365c-414b-8dbb-5c0cdd6a46b8", + "title": "CodeCommit", + "properties": { + "id": { "type": "string", "description": "commit sha" }, + "repo": { "type": "string", "description": "owner/name" }, + "ref": { "type": "string", "description": "branch the push landed on" }, + "message": { "type": "string" }, + "authorEName": { "type": "string" }, + "committedAt": { "type": "string", "format": "date-time" }, + "added": { "type": "array", "items": { "type": "string" } }, + "removed": { "type": "array", "items": { "type": "string" } }, + "modified": { "type": "array", "items": { "type": "string" } }, + "diffUrl": { "type": "string", "description": "the diff's own S3 URL — see below, never inlined" } + }, + "required": ["id", "repo", "ref", "message", "authorEName", "committedAt", "diffUrl"] +} +``` + +**Revised design: the diff is always uploaded to S3, never inlined, and no size cap decides that — a size cap was the +original design, superseded after live testing found two things wrong with it.** The first draft capped diffs at a +configurable size (`FORGEJO_SYNC_DIFF_MAX_BYTES`), inlining under the cap and falling back to a link back to GitW3 +above it — mirroring `file.json`'s own `data`-vs-`url` split. Reviewed against an explicit requirement that the diff +must be preserved regardless of size — including sizes inlining was never going to handle — a cap-then-fallback +design doesn't fit: eVault's own GraphQL server caps request bodies at 350MB +(`infrastructure/evault-core/src/index.ts:184`, `bodyLimit: 350 * 1024 * 1024`), and a large blob doesn't belong +inlined into a graph-database node property well under that ceiling either — which is exactly why `file.json`, the +one other ontology schema in this codebase modelling large content, has a `url` field in the first place. So instead +of choosing a cap, every diff is uploaded to the same DigitalOcean Spaces (S3-compatible) bucket +`infrastructure/evault-core/src/services/StorageService.ts` already uses, and only the resulting URL is written into +the MetaEnvelope. **Uploaded directly, not through evault-core's own `uploadFile` GraphQL mutation** — that mutation +exists and is reachable on the same authenticated client this service already builds, but caps at 250MB +(`MAX_FILE_BYTES`) on top of the same 350MB body limit; going straight to S3 (same bucket, same `DO_SPACES_*` +credentials, no new secrets) has no such ceiling. + +**The diff-fetch endpoint was also wrong in the original draft, found only by testing against a live private repo, not +by re-reading source harder.** The draft cited `routers/web/web.go:1808`'s `GET /{owner}/{repo}/commit/{sha}.diff` — +real, and gated by `reqRepoCodeReader` in the source, which reads as "needs code-read access, so an admin token +should satisfy it." Tested directly against a running GitW3 instance: the exact same request, same token, three auth +forms tried (`Authorization: token`, HTTP Basic, `?token=`), returned `404` on a private repo and `200` on the same +repo made public. **The web router's `.diff` route does not authenticate a PAT for a private repo at all** — it +evaluates the request as anonymous regardless of credentials, and Forgejo denies anonymous access to a private repo +with `404` rather than `403`, to avoid revealing the repo exists. The fix, also confirmed live: fetch from the **API +router** instead — `GET /api/v1/repos/{owner}/{repo}/git/commits/{sha}.diff` +(`routers/api/v1/repo/commits.go`'s `DownloadCommitDiffOrPatch`, registered under `/api/v1/repos/{owner}/{repo}/git` +in `routers/api/v1/api.go`) — same diff content, but on the standard PAT-aware auth chain the admin token already +proves itself against for the Users API and diff-adjacent calls. Confirmed working on a live private repo with the +same token that 404'd on the web-router path. + +**The S3 object's own ACL must mirror the repo's visibility, for the same reason the envelope's ACL does.** Uploading +every diff `public-read` regardless of source-repo visibility would recreate, one layer down, exactly the problem +`deriveAcl` exists to avoid: a private repo's diff would be readable by anyone with the URL, independent of whatever +ACL the eVault envelope itself carries. `content/diff.ts`'s upload call is `public-read` only when +`!task.repoPrivate`; a private repo's diff is uploaded with no public ACL, so its URL is not fetchable without the +bucket's own credentials. There is no presigned-URL-on-read feature built for this — out of scope for this pass; +retrieving a private diff later needs direct bucket access, not a link a browser can just open. **Not fully verified +end to end**: the request correctly carries the ACL header (confirmed against a real S3-compatible server, not just +mocked), but the specific test environment used to check it (a local MinIO instance) doesn't honour legacy per-object +ACLs the way DigitalOcean Spaces does — this is a known difference between MinIO's and AWS/DO's S3 implementations, +not a sign the request is wrong, but it means the actual public/private *enforcement* has only been proven against +evault-core's own already-working use of this same pattern against real DO Spaces, not independently re-verified for +this service's own uploads. + +## Repo-owner full snapshot (added 2026-08-15) + +Everything above this section is the per-pusher commit+diff sync, unchanged by what follows. This section adds a +second, independent sync path off the same webhook, built after the first was already live-verified: the repository +**owner**'s eVault gets the complete repo — every file and folder, not a diff — stored via S3 and **replaced in +place** on every push. In the owner's own words: "the owner of the repo has the whole repo stored and which just +replaces whenever anyone makes a commit. And for the individual contributors, the commit along with diff gets stored +in their evaults." Both paths run off the same `push` webhook delivery; neither is disabled or weakened by the +other, and one succeeding, skipping, or failing has no bearing on the other's outcome for the same push. + +**Owner, not pusher.** `repository.owner.login` (`modules/structs/repo.go:52`, `Owner *User \`json:"owner"\``) is a +full `User`, same struct as `pusher` — but that is not assumed just because both are `*User`; `modules/structs/user.go:18` +confirms `UserName string \`json:"login"\`` is the same tag pusher.login already uses. Resolved to an eName with the +exact same `IdentityResolver` the pusher path already uses (`identity.ts` was already generic over any Forgejo +username, not pusher-specific — no new identity code needed). A repo pushed to by a collaborator has an owner who +never touched `git push` for that commit at all; the owner-snapshot sync doesn't care who pushed, only who owns the +repo. + +**Org-owned repos fall out of identity resolution for free, with no separate check.** `GET /api/v1/users/{username}` +has no user-type filter (`routers/api/v1/user/user.go`'s `GetInfo` — checked directly, gates only on +`IsUserVisibleToViewer`, not on the account being a person rather than an organization; Forgejo stores organizations +as rows in the same user table). So calling it with an organization's login succeeds, but an organization never signs +in through the bridge, so its `login_name` never starts with `@` — `enameFromLoginName` returns `null` for it exactly +the way it does for an ordinary unlinked password account. **Treated identically, deliberately, not by oversight**: +an org-owned repo's snapshot sync is skipped, logged the same "no linked eVault" way a personal unlinked account's +push is. + +**Update in place, not one envelope per push.** "Replaces whenever anyone makes a commit" means one `repoSnapshot` +MetaEnvelope per repo, kept current via `updateMetaEnvelope(id: ID!, input: MetaEnvelopeInput!): UpdateMetaEnvelopePayload!` +(`infrastructure/evault-core/src/core/protocol/typedefs.ts:335`, resolver at `graphql-server.ts:466`) rather than a +fresh `createMetaEnvelope` on every push. `evault-core` also has a legacy `updateMetaEnvelopeById(id: String!, ...)` +mutation (`typedefs.ts:368`) — not used here; the "new" `updateMetaEnvelope` matches `createMetaEnvelope`'s own +idiomatic payload shape (`{ metaEnvelope { id }, errors { field message code } }`) this service's `writeCommit` +already relies on, so `EVaultClient.writeRepoSnapshot` could reuse the same error-unwrapping code for both mutations. +Finding "the existing envelope id for this repo" on a later push needed somewhere to remember it — deliberately +**not** `evault-core`'s `metaEnvelopes` list query, which has the confirmed, live, currently-unfixed ACL-filtering +bug this doc's Testing section already documents. Using it to search for this service's own bookkeeping wouldn't hit +that bug in a security-relevant way (this service already knows the eName and repo it's looking for), but it's an +unnecessary live dependency on a component with a known correctness bug when a local mapping is simpler and already +matches `queue.ts`'s own file-per-key persistence style — see `repoEnvelopeStore.ts`. + +**Once per push, not once per commit.** A 10-commit push must upload the repo once, not ten times. The webhook +handler builds at most one `RepoSnapshotTask` per delivery, outside the per-commit loop that builds `CommitSyncTask`s, +using the push's own `after` field (`modules/structs/hook.go:261`, `After string \`json:"after"\`` — the sha the ref +points at once the push lands) as the archive's ref, not any individual commit's id — a multi-commit push has several +of those, and only the final one is "the repo's current state" worth archiving. `after` being the all-zero sha (a +branch/tag deletion) skips queuing a snapshot task entirely — there is no ref state left to archive. + +**The archive endpoint — found and live-verified, not assumed, same discipline that caught the web-router `.diff` +bug.** `routers/api/v1/api.go:847`: `m.Get("/archive/*", reqRepoReader(unit.TypeCode), repo.GetArchive)`, mounted +under `m.Group("/repos", ...) { m.Group("/{username}/{reponame}", ...) }` (line 780) — i.e. +`GET /api/v1/repos/{owner}/{repo}/archive/{ref}.{zip|tar.gz|...}`, the API router, never the web router's own +separate `/archive` group in `routers/web/web.go:1916`, which this service does not use at all. Confirmed live against +the real private test repo (`2086ed05-.../forgejo-code-sync-test`), not just read from source: `curl` with +`Authorization: token ` → `200` with a real zip archive (`unzip -l` showed the repo's actual files and +folders); the identical request with no `Authorization` header → `404` (Forgejo's usual anonymous-on-private-repo +response, not `403`, so as not to leak that the repo exists) — the same auth behaviour as the working `.diff` route, +not the broken one. Also confirmed with a real commit sha as the ref (not just a branch name), since that's what +`RepoSnapshotTask.headCommitId` always is: `GET .../archive/8e944efc31f3ff176901ceed8ee2f4fe49ccc42d.zip` → `200`, +same content as `.../archive/main.zip` at that point in the repo's history. `.zip` was picked over `.tar.gz` (both +were confirmed working) for no reason beyond it being marginally more universal to extract; nothing about the design +depends on the choice. + +**S3 key and ACL — one object per repo, mirrors visibility the same way diffs already do.** `S3Storage.uploadRepoArchive` +uploads to a deterministic, collision-free key `repos/{owner}/{repo}.zip` — one per repo, not one per commit or push, +the same key reused (and overwritten) on every later push, which is what makes "update in place" true at the S3 layer +too, not just the eVault layer. `isPublic` mirrors `!task.repoPrivate`, the exact same signal `uploadDiff` already +uses, for the exact same reason: a private repo's full source must not become world-readable via a guessable S3 URL +just because it's stored as an archive rather than a diff. Same accepted, named limitation as the diff path: if a +repo's visibility changes between pushes, the already-uploaded archive's ACL does not retroactively change — the next +push re-uploads (and re-ACLs) it, but nothing re-scans an already-synced repo that hasn't been pushed to again. + +**Live-verified end to end**, against the same local MinIO throwaway container and the same `mc anonymous` +differential-proof technique the diff path's own 2026-08-15 verification note (below, in Testing) used — see that +section for the full commands and output. Summary: a real two-push sequence against a real public test repo produced +one `repoSnapshot` envelope (checked directly against Neo4j) whose `id` was identical across both pushes and whose +`headCommitId`/`snapshotUrl` content updated to match the second push's HEAD; a real push against a real private test +repo produced an owner-only-ACL'd envelope whose S3 archive was anonymously denied while the public repo's stayed +anonymously fetchable, at the same bucket, same host, same container; a push from an unlinked, password-registered +account produced zero envelopes on either sync path, logged as two distinct "skipped" lines, not an error. + +## Architecture + +Same shape as the bridge: `services/forgejo-code-sync/`, flat, Express + TypeScript, `.env`-driven required-config +pattern (`src/config.ts`, throws at startup on anything missing — mirrors +[awareness-service's config.ts](../../../services/awareness-service/api/src/config.ts) and the bridge's own). + +``` +config.ts env parsing; throws at startup on anything missing +identity.ts pusher/owner username -> eName, admin API call + TTL cache - generic, used by both sync paths +evault.ts certify + per-eName GraphQL client (copy of EVaultService.ts's shape), acl derived from Repo.Private + writeCommit (create-only) and writeRepoSnapshot (create-or-update) both live here +storage/s3.ts uploads a diff or a repo archive to the same DO Spaces bucket evault-core uses, ACL mirrors repo + visibility - uploadDiff keyed per commit, uploadRepoArchive keyed per repo (overwritten in place) +content/diff.ts fetch a commit's diff from the API router, upload it via storage/s3.ts, return the S3 URL +content/archive.ts fetch a repo archive from the API router at the push's final ref, upload it, return the S3 URL +repoEnvelopeStore.ts repoFullName -> envelopeId, so a later push updates the same repoSnapshot envelope in place +queue.ts persisted retry queue — see Delivery reliability, below - one instance per task kind +sync.ts per-commit drain loop (pusher's eVault) +snapshotSync.ts once-per-push drain loop (owner's eVault) - mirrors sync.ts's shape and skip/retry semantics +webhook/push.ts verify signature, iterate commits (per-commit tasks), build one snapshot task per delivery +index.ts wiring, /healthz +``` + +``` + GitW3 (push) forgejo-code-sync Forgejo API eVault (pusher's) + │ │ │ │ + ├─ POST /webhook ─────▶│ │ │ + │ X-Forgejo-Signature │ │ │ + │ ├ verify HMAC │ │ + │ ├ persist delivery to queue │ │ + │◀── 200 (queued) ─────┤ │ │ + │ │ │ │ + │ (async, per commit, retried on failure — see below) │ + │ ├ pusher.username cached? ────┤ GET /users/:name │ + │ │◀─────────────────────────── login_name │ + │ ├ login_name starts with @? ─┘ else: skip, dequeue │ + │ ├ fetch diff, upload to S3 ───────────────────────────▶ S3 + │ ├ certify (cached) ──────────────────────────────────▶ + │ ├ acl = Repo.Private ? owner-only : ["*"] │ + │ ├ createMetaEnvelope(codeCommit, X-ENAME, acl) ─────▶ + │ │◀───────────────────────────────────────── envelope id + │ ├ dequeue on success │ +``` + +The webhook handler's own response is now decoupled from whether the sync actually succeeds — it acknowledges receipt +once the delivery is durably queued, and the queue drains asynchronously with its own retry policy. This matters +because of what [Delivery reliability](#delivery-reliability-no-safety-net-from-forgejo) below found: Forgejo will +never redeliver a failed webhook on its own, so responding `200` only after a successful eVault write would just +convert a transient failure into permanent silent loss, indistinguishable from the "no linked eVault" skip case. + +One webhook delivery can carry several commits (a multi-commit push); each is queued and processed independently, so +a partial failure (one commit's diff fetch fails) doesn't block the rest. + +### Delivery reliability: no safety net from Forgejo + +**GitW3-verified, and it changes the reasoning here**: Forgejo has **no automatic retry/redelivery** of failed +webhook deliveries. `services/webhook/deliver.go` records `t.IsSucceed = resp.StatusCode/100 == 2` and +`w.LastStatus`, then stops — there is no requeue/retry logic anywhere in `services/webhook` (checked directly, none +found). The only resend path is a human clicking "Replay" on a specific delivery in the repo's webhook history UI +(`templates/repo/settings/webhook/history.tmpl`, `POST .../replay/{UUID}`) — nothing automatic, nothing that +re-delivers a whole batch on its own. + +**Decision: this service owns its own reliability, rather than accepting best-effort loss.** A timeout, an eVault +outage, or a crash mid-batch must not silently drop a push the way an unlinked account silently skips one — those are +different situations (one is expected and permanent, the other is transient and should resolve on retry) and must not +look the same from the outside. `queue.ts` persists each commit-sync task before attempting it, retries with backoff +on failure, and distinguishes — in logs/metrics, not just internally — "skipped, no linked eVault" from "failed, +retrying" from "failed, retries exhausted, needs attention." The last category is the one that needs a real alert; +silently swallowing it would reproduce exactly the invisible-data-loss failure mode this decision exists to avoid. + +## Trust model + +**HMAC verification is the only thing standing between this service and an attacker POSTing a fabricated push.** +Same posture as the bridge's `client_secret` check: constant-time comparison, secret never logged, raw body bytes +hashed rather than a re-serialized `req.body` — see the concrete implementation under +[Forgejo webhook side](#forgejo-webhook-side). + +**The admin token is the new, larger risk this design adds, and it is bigger than the original draft assumed.** The +bridge's signing key can forge an identity; this token can *read* every account's `login_name` and whatever else +`GET /api/v1/users/{username}` returns to an admin — broader blast radius than the bridge needed. + +**GitW3-verified, two corrections to the token sizing:** + +1. **The exact scope is `read:user`** (`models/auth/access_token_scope.go:83`, category + `AccessTokenScopeCategoryUser`) — that part narrows cleanly. But **scope is not what gates `login_name`; the + account's admin flag is.** `services/convert/user.go`'s `toUser` only fills in `LoginName` when + `authed = doer.ID == user.ID || doer.IsAdmin` (lines 24, 77-85). A `read:user`-scoped PAT belonging to a + *non-admin* account will call `GET /users/{username}` successfully but get back `login_name: ""` for anyone but + itself — there is no scope that substitutes for the account actually being a site admin. So "narrowest scope" and + "must be a site-admin's token" are both true and independent constraints. +2. **This same token also needs to cover diff fetching** (see the `.diff` note under + [What gets written](#what-gets-written)) — the commit-diff route requires `read:repository` + (`services/context/permission.go:65-76` rejects a scoped token missing it) plus `ctx.Repo.CanRead`, which an + admin account satisfies for any repo regardless of collaborator status. Practically: **one PAT, scopes + `read:user,read:repository`, on a site-admin account.** + +**Decision: a dedicated site-admin service account, not a human admin's personal token.** Given the token's blast +radius — it can read `login_name` and repo content for every account and every repo on the instance, not just what +this service needs at any given moment — it is provisioned as its own account, created for this purpose alone, not +borrowed from whoever happens to administer GitW3. It is stored and rotated with the same care as the bridge's +`W3DS_OIDC_SIGNING_KEY`: never committed, never logged, and treated as a high-value secret in whatever secret store +the eventual deployment uses. This does not shrink the token's actual capability — Forgejo's permission model doesn't +offer anything narrower that still satisfies `login_name` and private-repo diff access — so the mitigation is +isolation and custody, not scope reduction. + +**Decision: ACL on the written envelope mirrors the repo's visibility, not a constant.** +`EVaultService.ts` writes `acl: ["*"]` unconditionally because a calendar event is meant to be readable by whoever +the person shares it with via the platform. Code from a private repository is not that — defaulting to `["*"]` here +would make private source world-readable the moment it's synced, independent of GitW3's own visibility setting. So +this service conditions the write on the signal already present in every push payload at no extra cost: +`Repository.Private` (`modules/structs/repo.go:57`, confirmed present). `acl: [eName]` when the repo is private at +push time, `acl: ["*"]` when public — `[eName]` confirmed as the codebase's own convention for a restricted ACL, the +one precedent being `infrastructure/evault-core/src/services/BindingDocumentService.ts:298,378`'s +`acl: [normalizedSubject]`; every other write anywhere in the codebase uses `["*"]` with no exception. + +**Known limitation, confirmed against `evault-core`'s own access-control code, not assumed: "owner-only" here means +"not public," not "restricted to the owner."** `vault-access-guard.ts`'s `checkAccess` — the resolver path for a +single envelope fetched by ID — grants access to **any request carrying a valid Registry-issued Bearer token from any +certified platform**, without consulting the envelope's `acl` at all in that branch; the ACL is only actually checked +against an anonymous request (no valid token) or inside the bulk `metaEnvelopes` list query, which has no such +bypass. So `acl: [eName]` reliably keeps a private-repo commit out of anonymous reach and out of another platform's +list-query results, but does **not** stop a different certified platform from reading the same envelope directly by +ID if it already has the ID and the right `X-ENAME` — there is nothing this service can do about that without +patching `evault-core` itself, which is out of scope here. Documented rather than fixed: `acl: [eName]` is still +strictly better than `acl: ["*"]` (it removes the public and cross-platform-browsing exposure), and matches the one +real precedent for restricted data in this codebase, but it is not the airtight privacy guarantee the word +"owner-only" might otherwise suggest. + +**Known limitation, accepted rather than solved: visibility is captured at push time, not kept in sync afterward.** +If a repo is public when a commit is pushed — so its envelope is written `acl: ["*"]` — and is later flipped private +on GitW3, the already-synced envelope stays world-readable. This service has no trigger for a later visibility change +and does not re-scan already-synced commits. The symmetric case (private → public) similarly does not retroactively +open up envelopes synced while private. Closing this gap would require either polling every synced repo's current +visibility on some schedule or GitW3 emitting a visibility-change event, and neither exists today — this is a real, +named gap, not an oversight. + +**Transport.** Both the inbound webhook and the outbound admin-API call need TLS in any environment beyond local dev, +for the same reason as the bridge: the webhook secret and the admin token are both bearer credentials with no other +protection in transit. + +## Deployment + +Inherits the bridge's own unresolved blocker: **no service deployment manifest exists in this repository for a +production or staging host.** `docker-compose.gitw3.yml` is explicitly a candidate, not a convention. This spec +doesn't attempt to resolve that a second time — whatever answer the bridge's deployment gets, this service follows. +That decision was reaffirmed in review rather than revisited: it doesn't block writing the implementation plan or the +code, only an actual staging rollout, matching how the bridge's own spec left it. + +What's specific to this service, once a host is known: + +- `docker/Dockerfile.forgejo-code-sync`, following the `docker/Dockerfile.` convention. +- The system webhook can be scripted, but not via CLI. **GitW3-verified**: `cmd/` has no webhook subcommand at all + (checked — nothing under `cmd/*.go` matches), so this is not analogous to how + [`docker/gitw3-register-auth-source.sh`](../../../docker/gitw3-register-auth-source.sh) scripts the auth source via + `gitea admin auth add-oauth`. It doesn't need to be: the Admin REST API already does this — + `POST /api/v1/admin/hooks` (`routers/api/v1/admin/hooks.go`'s `admin.CreateHook`, mounted in `routers/api/v1/api.go` + around line 1365) accepts a `CreateHookOption` body and creates exactly the "system webhook" Site Administration's + UI creates, scoped instance-wide. So this is scriptable — a small `curl`/script against that endpoint with an + admin token, run once at provisioning time — just via the API instead of a CLI subcommand. +- The retry queue (see [Delivery reliability](#delivery-reliability-no-safety-net-from-forgejo)) needs somewhere to + persist pending deliveries across a restart — not necessarily a new database if this service ends up sharing + infrastructure with something else already deployed, but not nothing either. Left open pending the same + deployment-path answer as everything else in this section. + +| Variable | Note | +|---|---| +| `FORGEJO_SYNC_PUBLIC_URL` | this service's own base URL, used for Registry platform certification | +| `FORGEJO_SYNC_PORT` | | +| `FORGEJO_WEBHOOK_SECRET` | HMAC secret configured on the Forgejo system webhook | +| `FORGEJO_API_URL` | GitW3's base URL, for the admin Users API call | +| `FORGEJO_ADMIN_TOKEN` | PAT on a **dedicated site-admin service account** created for this service alone (not a shared human admin's token), scopes `read:user,read:repository` — see [Trust model](#trust-model) for why both scopes and the admin flag are required | +| `PUBLIC_REGISTRY_URL` | already in the root `.env` | +| `PUBLIC_EVAULT_SERVER_URI` | already used by the calendar platform's `EVaultService.ts` | +| `DO_SPACES_ENDPOINT`, `DO_SPACES_REGION`, `DO_SPACES_KEY`, `DO_SPACES_SECRET`, `DO_SPACES_BUCKET`, `DO_SPACES_CDN_URL` | already in the root `.env` — the same bucket `infrastructure/evault-core/src/services/StorageService.ts` uses. No service-specific S3 credentials needed | + +## Testing + +Same split as the bridge: pure logic first, wallet/Forgejo-dependent behaviour second. + +**Unit — `identity.ts`.** `login_name` starting with `@` resolves; one that doesn't is treated as "no eVault," not an +error; cache hit skips the API call; a 404 evicts the cache entry. + +**Unit — signature verification.** Valid HMAC accepted; one byte flipped in the body rejected; missing header +rejected; a `sha256=`-prefixed value naively compared against the unprefixed header rejected as a regression guard +for the trap above — same shape as the bridge's `client_secret` tests. + +**Unit — `content.ts`.** Fetches from the API router's `git/commits/{sha}.diff`, not the web router's +`commit/{sha}.diff` — a regression guard specifically for the trap above. Uploads with `public-read` for a public +repo, no public ACL for a private one. Throws — does not degrade to a fallback — on a fetch failure, a non-2xx +response, or an S3 upload failure, since there is no longer a lesser alternative to fall back to. + +**Unit — `evault.ts` ACL derivation.** `Repo.Private: true` produces an owner-only `acl`; `false` produces `["*"]`; +the derivation is a pure function of the payload, independent of the identity lookup. + +**Unit — `queue.ts`.** A task that fails is retried with backoff, not dropped; one that exhausts retries is marked +distinguishably from one that's still pending, and distinguishably from an ordinary "no eVault" skip; a queued task +survives a process restart (persisted, not in-memory only). + +**End to end, no real Forgejo webhook needed for the eVault half.** The Dev Sandbox provisions an eVault; combined +with the bridge's own local flow (sign into a local GitW3 via W3DS, which is what sets `login_name`), a real +`login_name` can be read and a synthetic webhook payload POSTed directly at this service to exercise the full chain +without needing a live push. + +**Done, not just planned: a real push against a real system webhook, on a live GitW3 instance with a real +W3DS-linked account.** Registered the system webhook via the provisioning script against a running GitW3 +(`v16.0.2-9`), pushed a real commit to a real repo (public, then flipped private) owned by an account already linked +through the bridge, and confirmed a `codeCommit` MetaEnvelope landed with the correct `acl` — checked directly +against Neo4j, not just the GraphQL read path (see the caveat about that read path below). This is what surfaced both +corrections in this section: the original size-cap design and the web-router diff endpoint. Two operational findings +from that run, not code defects: Forgejo's `ALLOWED_HOST_LIST` blocks a webhook targeting a loopback address by +default, which only bites when the target happens to be `localhost` relative to GitW3 (true in this local setup, not +expected to be true of a real deployment — see [Deployment](#deployment)); and registering more than one webhook +pointed at this service's URL produces one envelope per delivery received, since the service has no reason to assume +two separate deliveries describe the same event — an operational hazard (don't register it twice), not something the +service should paper over with deduplication it can't actually justify. + +**A gap found during that same live run, in `evault-core`, not this service, not yet acted on.** Querying +`metaEnvelopes` with a valid platform-certification token and *any* certified platform's identity — including one +that was never registered anywhere, just self-certified via the open `/platforms/certification` endpoint — returned +a private, owner-only-ACL'd envelope in full. Traced to the cause: `graphql-server.ts:237`'s `metaEnvelopes` resolver +returns a Relay-style connection object (`{edges, pageInfo, totalCount}`), not a bare array, so +`vault-access-guard.ts`'s `filterEnvelopesByAccess` — the actual ACL check — never runs for it; the middleware's +array-detection branch only applies to a bare array, and the connection-object branch (`filterACL`) only strips a +top-level `acl` field, which a connection wrapper doesn't have. **Practical effect: `acl: [eName]` currently provides +no protection at all against this specific query, for any certified platform.** This is `evault-core`'s bug, not +fixable from this service, and worse than the single-ID-lookup gap already documented below — raised here rather +than silently left for someone else to rediscover. + +**2026-08-15: the S3 ACL split, closed — and a fourth GitW3 webhook trap, found live.** The previous run above proved +the AWS SDK call succeeds with and without `ACL: public-read`, but not that anonymous access actually differs, because +the MinIO instance used didn't honour legacy per-object canned ACLs. This pass closed that gap and re-ran all four +live-push scenarios end to end against the corrected diff/S3 design. + +**S3 ACL split, proven for real this time.** Real DigitalOcean Spaces credentials weren't available for this pass (a +`decision_gate`-equivalent ask to the operator confirmed this before proceeding) — the fallback the spec already +called out was used instead: a fresh throwaway MinIO container (`quay.io/minio/minio:RELEASE.2025-04-22T22-12-26Z`, +port `9010`, started with `MINIO_DOMAIN=localhost` so its virtual-hosted-style bucket resolution matches what +`S3Storage`'s `forcePathStyle: false` client actually sends — without that env var MinIO parses the first path +segment as the bucket name instead of the Host header, which is a MinIO configuration quirk, not a service bug). +`S3Storage.uploadDiff` — the service's real code path, not a hand-rolled `PutObjectCommand` — uploaded one object with +`isPublic: true` and one with `isPublic: false`. Immediately after upload, both were anonymously denied +(`curl -o /dev/null -w '%{http_code}'` → `403` for both), reproducing the exact "MinIO doesn't honour legacy +per-object ACLs" limitation this section already named — the AWS SDK call succeeding is not the same as the object +being open. Only then was `mc anonymous set download local///` run, scoping the open +policy to exactly the prefix the public-repo upload used and leaving the private-repo prefix untouched — the closest +local substitute for what a real per-object canned ACL does on DO Spaces. A second anonymous `curl` after that: the +public object returned `200` with its real diff content; the private object, at the same bucket, same host, same +container, was still `403 AccessDenied`. This is a genuine differential proof, not two upload calls both succeeding. + +**Four live pushes, all four re-verified against the corrected (S3-always, API-router-diff) design, checked directly +against Neo4j, not the GraphQL read path** (see the `metaEnvelopes` gap above — still open, still avoided here): + +- **Public repo.** Pushed to a fresh throwaway repo (`2086ed05-.../forgejo-code-sync-test-public`, created public) + owned by the W3DS-linked test account. `MATCH (m:MetaEnvelope {id:'bb2d7f57-f539-54df-b87f-75616d7a9843'})` returned + exactly one node, `acl: ["*"]`, `eName: "@2086ed05-..."`, linked `Envelope` nodes carrying the right `repo`, commit + `id`, and a `diffUrl` pointing at the MinIO bucket above. `curl`ing that `diffUrl` anonymously, after opening only + its prefix per the ACL proof above, returned `200` with the exact diff text the push produced. A second query, + `MATCH (m:MetaEnvelope)-[:LINKS_TO]->(:Envelope {ontology:'id', value:'a460119b...'}) RETURN count(m)`, returned + `1` — one envelope for one commit, not three. +- **Private repo.** Pushed to `2086ed05-.../forgejo-code-sync-test` (already private). Its envelope + (`a1e2fe77-b81e-5be9-a055-45199f45157b`) carries `acl: ["@2086ed05-a045-574f-a9e1-2ed1cf44bd75"]` — owner-only, not + `["*"]` — confirmed the same way, directly against Neo4j. Its `diffUrl`, anonymously curled with no policy applied + to that prefix, returned `403 AccessDenied`; the same object, fetched with the bucket's own credentials + (`mc cat local//`), returned the real diff content. Both properties hold on the same object at once: + fetchable with credentials, denied without them. +- **Unlinked account, regression-checked.** Created a plain password-registered GitW3 account + (`./gitea admin user create --username plain-test-user ...`, no OAuth2 source). `GET /api/v1/users/plain-test-user` + with the admin token omits the `login_name` key from the response JSON entirely — it is not present as `""`. This + service's own `IdentityResolver.resolveEname` already handles that correctly (`body.login_name ?? ""` at + `src/identity.ts:100`), but every existing unit test only exercised the `{ login_name: "" }` shape, never the + key-omitted-entirely shape a real GitW3 actually sends for this account type — a real gap the live response + surfaced, not a code defect. Fixed by adding a regression test (`src/identity.test.ts`, + "returns null when login_name is absent from the response entirely, not just empty") pinned to that exact shape. + Pushing a commit from this account logged `skipped - no linked eVault for pusher "plain-test-user"` — distinct from + an error — no `MetaEnvelope` was written (`MATCH (e:Envelope {ontology:'id', value:'5114c29...'}) RETURN count(e)` + → `0`), and the on-disk queue directory was empty afterward, not holding a stuck or exhausted task. +- **Exactly one webhook delivery per push.** `GET /api/v1/admin/hooks` showed exactly one hook throughout this pass, + and each push above produced exactly one `MetaEnvelope` (counted directly against Neo4j, per above) — the + three-envelopes-for-one-commit regression from the earlier stray-webhook incident did not reappear. + +**A fourth GitW3 trap, found only by rotating the webhook secret live, not by re-reading source harder.** +`scripts/register-webhook.ts`'s "idempotent, updates on redeploy" design (Phase 5.4) assumed `PATCH +/admin/hooks/{id}` could rotate `config.secret` the same way `POST /admin/hooks` sets it on create. It cannot: +`routers/api/v1/utils/hook.go`'s `editHook` (the function `PATCH /admin/hooks/{id}` calls) updates `url`, +`content_type`, `events`, `branch_filter`, and the authorization header from the request's `config` map, but never +reads `config["secret"]` anywhere in the function — only `addHook` (the `POST` path, line 187, +`Secret: form.Config["secret"]`) does. Reproduced directly: PATCHing the existing hook with a new +`FORGEJO_WEBHOOK_SECRET` returned `200`, `GET /admin/hooks` showed nothing wrong (Forgejo never echoes a hook's +secret back, by design, so there was nothing to check against), and the next real push's delivery came back with +`is_succeed = 0` and `response_content` `{"status":401,...,"body":"{\"error\":\"invalid signature\"}"}` in +GitW3's own `hook_task` table — confirmed by computing the HMAC of the actual delivered body with the *new* secret +and finding it did not match the signature GitW3 actually sent. Exactly the same silent-success shape as the three +traps this section already documents (`active` defaulting false, the `is_system_webhook` string field, the +web-router `.diff` endpoint): a `200`/`201` at provisioning time with the real failure only surfacing on the next +delivery, minutes or days later. **Fixed in `scripts/register-webhook.ts`**: rotating an existing hook's secret now +deletes it (`DELETE /admin/hooks/{id}`, confirmed to work for a real system hook despite `DeleteHook`'s handler being +named `DeleteDefaultSystemWebhook`) and recreates it via `POST`, rather than `PATCH`ing in place — the only way to +actually reach `addHook`'s secret-setting branch on a rotation, not just an initial create. + +**Regression suite after this pass: 90 unit tests (was 89) — `pnpm --filter forgejo-code-sync test` — and +`pnpm --filter forgejo-code-sync check` (Biome + `tsc --noEmit`, including `scripts/`) both still clean.** + +**2026-08-15, later the same day: the repo-owner full snapshot (see [that section](#repo-owner-full-snapshot-added-2026-08-15)), +built and live-verified end to end.** Same local MinIO container and `mc anonymous` technique as the diff-path note +above, reused rather than re-derived. + +**Archive endpoint, found and verified live before any code was written against it** — +`GET /api/v1/repos/{owner}/{repo}/archive/{ref}.zip`: + +``` +$ curl -sS -o /dev/null -w '%{http_code}' -H "Authorization: token " \ + 'http://localhost:3001/api/v1/repos/2086ed05-.../forgejo-code-sync-test/archive/main.zip' +200 +$ curl -sS -o /dev/null -w '%{http_code}' \ + 'http://localhost:3001/api/v1/repos/2086ed05-.../forgejo-code-sync-test/archive/main.zip' # no auth header +404 +$ unzip -l archive-priv-auth.zip + Archive: archive-priv-auth.zip + 8e944efc31f3ff176901ceed8ee2f4fe49ccc42d + Length Name + 0 forgejo-code-sync-test/ + 26 forgejo-code-sync-test/README.md + 66 forgejo-code-sync-test/hello.txt + 49 forgejo-code-sync-test/private-verify.txt +``` + +Same result requesting the archive by commit sha instead of branch name (`.../archive/8e944efc....zip`), which is +what `RepoSnapshotTask.headCommitId` always uses. `.tar.gz` confirmed working identically; `.zip` used going forward. + +**First push — envelope created, checked directly against Neo4j:** + +``` +[snapshot] 2086ed05-.../forgejo-code-sync-test-public@888848984147 -> envelope 55056a9c-4a7e-576d-883e-378da54016c2 + +neo4j> MATCH (m:MetaEnvelope {id:'55056a9c-4a7e-576d-883e-378da54016c2'}) RETURN m +(:MetaEnvelope {acl: ["*"], eName: "@2086ed05-...", ontology: "a9b56118-ac82-4f4e-9f70-77444c1a8f34"}) +# linked Envelope nodes: repo, ref, headCommitId (matches the pushed commit sha), ownerEName, snapshotUrl, updatedAt + +$ mc cat 'local/forgejo-sync-test/repos/2086ed05-.../forgejo-code-sync-test-public.zip' | unzip -l - + forgejo-code-sync-test-public/README.md + forgejo-code-sync-test-public/public-verify.txt + forgejo-code-sync-test-public/snapshot-verify.txt # the file the verification commit itself added +``` + +**Second push — same envelope id, content updated, not a second envelope:** + +``` +[snapshot] 2086ed05-.../forgejo-code-sync-test-public@0a3991204e55 -> envelope 55056a9c-4a7e-576d-883e-378da54016c2 +# ^^^^^^^^ identical to the first push + +neo4j> MATCH (m:MetaEnvelope {ontology:'a9b56118-...', eName:'@2086ed05-...'})-[:LINKS_TO]-> + (e:Envelope {ontology:'repo'}) WHERE e.value = '2086ed05-.../forgejo-code-sync-test-public' + RETURN count(m) +1 +# headCommitId on that same envelope now reads the second push's sha; snapshotUrl is byte-identical (same S3 key) + +$ mc cat '.../forgejo-code-sync-test-public.zip' | unzip -p - .../snapshot-verify.txt +repo-snapshot verification push 1 - 2026-08-14T19:47:38Z +repo-snapshot verification push 2 - 2026-08-14T19:48:31Z # both lines present - the object was overwritten, not appended, and the diff-fetch content itself proves it's live +``` + +**S3 ACL split, the same differential proof as the diff path, on the repo archive this time:** + +``` +# before opening any policy - both denied, MinIO's usual non-honouring of canned ACLs, expected +$ curl -o /dev/null -w '%{http_code}' '.../repos/.../forgejo-code-sync-test-public.zip' # 403 +$ curl -o /dev/null -w '%{http_code}' '.../repos/.../forgejo-code-sync-test.zip' # 403 (private repo) + +$ mc anonymous set download 'local/forgejo-sync-test/repos/2086ed05-.../forgejo-code-sync-test-public.zip' + +# after - only the public repo's archive opened +$ curl -o /dev/null -w '%{http_code}' '.../repos/.../forgejo-code-sync-test-public.zip' # 200, real zip content +$ curl -o /dev/null -w '%{http_code}' '.../repos/.../forgejo-code-sync-test.zip' # still 403 +``` + +**Unlinked owner (and, by the same mechanism, an org-owned repo — see the design section above for why they resolve +identically) — both sync paths skip cleanly, distinctly from an error, and write nothing:** + +``` +[snapshot] plain-test-user/plain-test-repo@11c08c157543 skipped - no linked eVault for owner "plain-test-user" +[sync] plain-test-user/plain-test-repo@11c08c157543 skipped - no linked eVault for pusher "plain-test-user" + +neo4j> MATCH (e:Envelope) WHERE e.value CONTAINS 'plain-test' RETURN count(e) +0 +``` +`.queue` and `.queue-snapshots` both empty afterward - neither task left pending, retrying, or exhausted. + +**Regression suite after this pass: 123 unit tests (was 90) — `pnpm --filter forgejo-code-sync test` — and +`pnpm --filter forgejo-code-sync check` (Biome + `tsc --noEmit`) both still clean.** + +**Staging / real Forgejo.** The local run above substitutes for most of what this note originally asked for. What's +still genuinely staging-only: TLS termination, a site-admin service account that isn't also someone's personal login, +and confirming the whole chain behaves the same once `forgejo-code-sync` and GitW3 are not on the same host. + +## Acceptance criteria + +| # | Criterion | Covered by | +|---|---|---| +| 1 | A push from a W3DS-linked GitW3 account writes commit records into that person's eVault | webhook → identity resolution → `createMetaEnvelope` | +| 2 | A push from an account with no linked eVault is skipped without error | `login_name` not starting with `@` → skip, dequeue | +| 3 | Commit authorship is never taken from unverified git commit metadata | identity resolved from `pusher`, never from `commit.author` | +| 4 | A commit's diff is preserved regardless of size | uploaded to S3, never inlined into the eVault write — see [What gets written](#what-gets-written) | +| 5 | Private-repo code is not made world-readable by the act of syncing it | `acl` derived from `Repo.Private` — see known limitation in [Trust model](#trust-model) | +| 6 | A transient failure (eVault outage, timeout, crash mid-batch) does not silently lose a push's sync | persisted retry queue — see [Delivery reliability](#delivery-reliability-no-safety-net-from-forgejo) | +| 7 | The repo owner's eVault holds the complete current repo, replaced in place on every push, independent of the pusher's own sync | [Repo-owner full snapshot](#repo-owner-full-snapshot-added-2026-08-15) — one `repoSnapshot` MetaEnvelope per repo, `updateMetaEnvelope` on every push after the first | + +## Open items + +None blocking implementation. Every item from the original draft was resolved during review — either by checking +directly against GitW3's actual pinned source (`v16.0.2`, not upstream Gitea by analogy; see the inline +"GitW3-verified" notes throughout and [Verification status](#verification-status) below), or by an explicit product +decision — ACL, admin-token custody, and delivery reliability, see [Trust model](#trust-model) and +[Delivery reliability](#delivery-reliability-no-safety-net-from-forgejo) — or, for `login_name`'s format, by chaining +three independently-verified facts across the bridge and GitW3 source (see +[Identity resolution](#identity-resolution-pusher--eName) and [Verification status](#verification-status)). + +Two things remain genuinely out of scope for this document rather than unresolved by it: + +- **Deployment path** — shared with the bridge's own unresolved blocker. An infrastructure/ownership question, not + something either repo's source can answer; does not block writing or running this service locally. +- **Where the retry queue persists** — depends on the same deployment answer above; noted under + [Deployment](#deployment) rather than repeated here. + +**The live smoke test this section previously called "still worth doing" has been done.** A real linked account, +signed in via the bridge on a live GitW3 instance, pushed real commits; the resulting envelopes and their ACLs were +checked directly against Neo4j. See [Testing](#testing) for what that run found — two design corrections (the diff +storage redesign and the web-router-vs-API-router diff endpoint) and one `evault-core` bug (the `metaEnvelopes` +list-query ACL gap), none of which surfaced from reading source alone. + +## Verification status + +Everything under [Identity resolution](#identity-resolution-pusher--eName) that cites `notifier.go`/`convert/user.go` +was checked directly against upstream Gitea source (`go-gitea/gitea`, `main` branch) during drafting, not assumed — +these are the two negative/positive findings this whole design depends on (ename absent from the webhook, present via +the admin API) and were the most important thing to get right before writing anything else. + +**Updated by a later review pass against GitW3's actual repository** (`/Users/sahil/orca/workspaces/gitw3`, pinned +`v16.0.2`, not upstream Gitea by analogy). Everything below was re-checked line-by-line against that source and is +now confirmed to hold on GitW3 specifically, superseding the "Gitea-sourced by analogy, not checked" caveat this +section originally carried for these items. **Independently spot-checked a second time**, directly against the same +GitW3 checkout, before folding these into the decisions above: `AddDefaultHeaders`'s unprefixed +`X-Forgejo-Signature`, the absence of retry/redeliver/requeue anywhere under `services/webhook`, the `read:user` +scope constant, `ToUser`'s admin-only `LoginName` gate, and `Repository.Private`'s presence on the payload struct all +match as claimed below. + +- `PushPayload`/`PayloadCommit`/`PayloadUser` struct shapes (`modules/structs/hook.go`) — match as drafted. +- `notifier.go`'s `PushCommits` calling `convert.ToUser(ctx, pusher, nil)` and `ToUser`'s `authed` gating + (`services/convert/user.go:16-27,49-87`) — match as drafted, including that `doer == nil` on the webhook path + makes `authed` always false, so `LoginName` is never in the webhook payload. +- `GET /users/{username}` populating `LoginName` only for an admin-or-self caller (`routers/api/v1/user/user.go:134`) + — confirmed, and the required scope is `read:user` exactly (`models/auth/access_token_scope.go:83`) — but scope + alone is not sufficient, the calling account must itself have `IsAdmin=true`. +- The `.diff`/`.patch` commit-URL suffix — real, at `routers/web/web.go:1808` → `repo.RawDiff`, gated by + `reqRepoCodeReader`, which for a scoped token additionally requires `read:repository`. +- `X-Forgejo-Signature`'s exact byte format (raw hex, unprefixed) vs. the `sha256=`-prefixed `X-Hub-Signature-256` + Forgejo sends alongside it (`services/webhook/shared/payloader.go`'s `AddDefaultHeaders`). +- System webhooks are scriptable via `POST /api/v1/admin/hooks` (`routers/api/v1/admin/hooks.go`) — no CLI + subcommand exists, but none is needed. +- Webhook delivery has **no automatic retry/redelivery** on failure anywhere in `services/webhook` — the only resend + is a human-triggered "Replay" in the web UI. +- OAuth2's `LoginName` assignment (`routers/web/auth/oauth.go:1141,1602`) uses `gothUser.UserID` verbatim in both the + new-account and existing-account lookup paths — consistent with the bridge doc's claim about what `login_name` + ends up holding. +- `Repository.Private bool` (`modules/structs/repo.go:57`) — confirmed present on the webhook's `Repo` field. + +Not re-checked in this pass (still analogy-only or out of GitW3's scope to verify): anything this service calls that +isn't one of the endpoints listed above. + +**The bridge's own `sub`-claim content, closed by a third pass, this time against the bridge's own source rather than +GitW3's.** `services/w3ds-oidc-bridge/src/claims.ts:171` — `buildClaims` returns `sub: ename` verbatim. The only +transformation applied to the ename anywhere in that file is `stripLeadingAt`, and it is used exclusively inside +`sanitiseUsername` (for `nickname`/`preferred_username`) and `emailLocalPart` — never on `sub`. So the full chain is +now confirmed across all three parties with no remaining analogy or assumption: + +``` +bridge: sub = ename, "@" included (claims.ts:171, this pass) +goth: UserID = ID token's "sub" claim (openidConnect.go, cited in the bridge's own design spec) +GitW3: LoginName = gothUser.UserID verbatim (oauth.go:1141,1602, previous pass) +──────────────────────────────────────────────────────────────────────────── + login_name on a linked account = the full, "@"-prefixed eName +``` + +This closes the one item the previous pass left as a live-instance-only check — it turned out answerable from source +on all three sides, so no running instance was required to hold it with confidence. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d3c4eab45..d272b3c9c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3272,7 +3272,7 @@ importers: version: 1.1.1(@types/react-dom@18.3.7(@types/react@18.3.27))(@types/react@18.3.27)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) draft-js: specifier: ^0.11.7 - version: 0.11.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + version: 0.11.7(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) lucide-react: specifier: ^0.561.0 version: 0.561.0(react@18.3.1) @@ -3293,7 +3293,7 @@ importers: version: 18.3.1(react@18.3.1) react-draft-wysiwyg: specifier: ^1.15.0 - version: 1.15.0(draft-js@0.11.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(immutable@5.1.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + version: 1.15.0(draft-js@0.11.7(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(immutable@5.1.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) react-hook-form: specifier: ^7.55.0 version: 7.71.2(react@18.3.1) @@ -4065,6 +4065,37 @@ importers: specifier: ^6.2.6 version: 6.4.1(@types/node@24.12.0)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2) + services/forgejo-code-sync: + dependencies: + '@aws-sdk/client-s3': + specifier: ^3.700.0 + version: 3.1009.0 + dotenv: + specifier: ^16.4.5 + version: 16.6.1 + express: + specifier: ^4.18.2 + version: 4.22.1 + graphql-request: + specifier: ^6.1.0 + version: 6.1.0(encoding@0.1.13)(graphql@16.13.1) + devDependencies: + '@types/express': + specifier: ^4.17.21 + version: 4.17.25 + '@types/node': + specifier: ^20.11.24 + version: 20.19.26 + tsx: + specifier: ^4.7.1 + version: 4.21.0 + typescript: + specifier: ^5.3.3 + version: 5.9.3 + vitest: + specifier: ^3.0.9 + version: 3.2.4(@types/debug@4.1.12)(@types/node@20.19.26)(@vitest/browser@3.2.4)(jiti@2.6.1)(jsdom@19.0.0(bufferutil@4.1.0))(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2) + services/ontology: dependencies: cors: @@ -30091,26 +30122,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@vitest/browser@3.2.4(bufferutil@4.1.0)(playwright@1.58.2)(vite@6.4.1(@types/node@20.19.26)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))(vitest@3.2.4)': - dependencies: - '@testing-library/dom': 10.4.1 - '@testing-library/user-event': 14.6.1(@testing-library/dom@10.4.1) - '@vitest/mocker': 3.2.4(vite@6.4.1(@types/node@20.19.26)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)) - '@vitest/utils': 3.2.4 - magic-string: 0.30.21 - sirv: 3.0.2 - tinyrainbow: 2.0.0 - vitest: 3.2.4(@types/debug@4.1.12)(@types/node@20.19.26)(@vitest/browser@3.2.4)(jiti@2.6.1)(jsdom@19.0.0(bufferutil@4.1.0))(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2) - ws: 8.19.0(bufferutil@4.1.0) - optionalDependencies: - playwright: 1.58.2 - transitivePeerDependencies: - - bufferutil - - msw - - utf-8-validate - - vite - optional: true - '@vitest/browser@3.2.4(bufferutil@4.1.0)(playwright@1.58.2)(vite@6.4.1(@types/node@22.19.15)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))(vitest@3.2.4)': dependencies: '@testing-library/dom': 10.4.1 @@ -30130,16 +30141,16 @@ snapshots: - utf-8-validate - vite - '@vitest/browser@3.2.4(bufferutil@4.1.0)(playwright@1.58.2)(vite@7.3.1(@types/node@22.19.15)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))(vitest@3.2.4)': + '@vitest/browser@3.2.4(bufferutil@4.1.0)(playwright@1.58.2)(vite@7.3.1(@types/node@20.19.26)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))(vitest@3.2.4)': dependencies: '@testing-library/dom': 10.4.1 '@testing-library/user-event': 14.6.1(@testing-library/dom@10.4.1) - '@vitest/mocker': 3.2.4(vite@7.3.1(@types/node@22.19.15)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)) + '@vitest/mocker': 3.2.4(vite@7.3.1(@types/node@20.19.26)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)) '@vitest/utils': 3.2.4 magic-string: 0.30.21 sirv: 3.0.2 tinyrainbow: 2.0.0 - vitest: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.15)(@vitest/browser@3.2.4)(jiti@2.6.1)(jsdom@19.0.0(bufferutil@4.1.0))(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2) + vitest: 3.2.4(@types/debug@4.1.12)(@types/node@20.19.26)(@vitest/browser@3.2.4)(jiti@2.6.1)(jsdom@19.0.0(bufferutil@4.1.0))(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2) ws: 8.19.0(bufferutil@4.1.0) optionalDependencies: playwright: 1.58.2 @@ -30259,6 +30270,15 @@ snapshots: optionalDependencies: vite: 6.4.1(@types/node@24.12.0)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2) + '@vitest/mocker@3.2.4(vite@7.3.1(@types/node@20.19.26)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))': + dependencies: + '@vitest/spy': 3.2.4 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 7.3.1(@types/node@20.19.26)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2) + optional: true + '@vitest/mocker@3.2.4(vite@7.3.1(@types/node@22.19.15)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))': dependencies: '@vitest/spy': 3.2.4 @@ -32730,9 +32750,9 @@ snapshots: dotenv@17.3.1: {} - draft-js@0.11.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + draft-js@0.11.7(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: - fbjs: 2.0.0 + fbjs: 2.0.0(encoding@0.1.13) immutable: 3.7.6 object-assign: 4.1.1 react: 18.3.1 @@ -32740,9 +32760,9 @@ snapshots: transitivePeerDependencies: - encoding - draftjs-utils@0.10.2(draft-js@0.11.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(immutable@5.1.5): + draftjs-utils@0.10.2(draft-js@0.11.7(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(immutable@5.1.5): dependencies: - draft-js: 0.11.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + draft-js: 0.11.7(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) immutable: 5.1.5 drizzle-kit@0.31.9: @@ -33193,8 +33213,8 @@ snapshots: '@typescript-eslint/parser': 5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2) eslint: 9.39.4(jiti@2.6.1) eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.6.1)) - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.6.1)) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)) eslint-plugin-jsx-a11y: 6.10.2(eslint@9.39.4(jiti@2.6.1)) eslint-plugin-react: 7.37.5(eslint@9.39.4(jiti@2.6.1)) eslint-plugin-react-hooks: 5.2.0(eslint@9.39.4(jiti@2.6.1)) @@ -33257,6 +33277,21 @@ snapshots: transitivePeerDependencies: - supports-color + eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)): + dependencies: + '@nolyfill/is-core-module': 1.0.39 + debug: 4.4.3(supports-color@5.5.0) + eslint: 9.39.4(jiti@2.6.1) + get-tsconfig: 4.13.6 + is-bun-module: 2.0.0 + stable-hash: 0.0.5 + tinyglobby: 0.2.15 + unrs-resolver: 1.11.1 + optionalDependencies: + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)) + transitivePeerDependencies: + - supports-color + eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@8.57.1): dependencies: '@nolyfill/is-core-module': 1.0.39 @@ -33299,6 +33334,17 @@ snapshots: - supports-color eslint-module-utils@2.12.1(@typescript-eslint/parser@5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)): + dependencies: + debug: 3.2.7 + optionalDependencies: + '@typescript-eslint/parser': 5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2) + eslint: 9.39.4(jiti@2.6.1) + eslint-import-resolver-node: 0.3.9 + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)) + transitivePeerDependencies: + - supports-color + + eslint-module-utils@2.12.1(@typescript-eslint/parser@5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.6.1)): dependencies: debug: 3.2.7 optionalDependencies: @@ -33338,7 +33384,7 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-plugin-import@2.32.0(@typescript-eslint/parser@5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.6.1)): + eslint-plugin-import@2.32.0(@typescript-eslint/parser@5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)): dependencies: '@rtsao/scc': 1.1.0 array-includes: 3.1.9 @@ -33367,6 +33413,35 @@ snapshots: - eslint-import-resolver-webpack - supports-color + eslint-plugin-import@2.32.0(@typescript-eslint/parser@5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.6.1)): + dependencies: + '@rtsao/scc': 1.1.0 + array-includes: 3.1.9 + array.prototype.findlastindex: 1.2.6 + array.prototype.flat: 1.3.3 + array.prototype.flatmap: 1.3.3 + debug: 3.2.7 + doctrine: 2.1.0 + eslint: 9.39.4(jiti@2.6.1) + eslint-import-resolver-node: 0.3.9 + eslint-module-utils: 2.12.1(@typescript-eslint/parser@5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.6.1)) + hasown: 2.0.2 + is-core-module: 2.16.1 + is-glob: 4.0.3 + minimatch: 3.1.5 + object.fromentries: 2.0.8 + object.groupby: 1.0.3 + object.values: 1.2.1 + semver: 6.3.1 + string.prototype.trimend: 1.0.9 + tsconfig-paths: 3.15.0 + optionalDependencies: + '@typescript-eslint/parser': 5.62.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.8.2) + transitivePeerDependencies: + - eslint-import-resolver-typescript + - eslint-import-resolver-webpack + - supports-color + eslint-plugin-jsx-a11y@6.10.2(eslint@8.57.1): dependencies: aria-query: 5.3.2 @@ -34089,7 +34164,7 @@ snapshots: fbjs-css-vars@1.0.2: {} - fbjs@2.0.0: + fbjs@2.0.0(encoding@0.1.13): dependencies: core-js: 3.48.0 cross-fetch: 3.2.0(encoding@0.1.13) @@ -34985,9 +35060,9 @@ snapshots: html-tags@3.3.1: {} - html-to-draftjs@1.5.0(draft-js@0.11.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(immutable@5.1.5): + html-to-draftjs@1.5.0(draft-js@0.11.7(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(immutable@5.1.5): dependencies: - draft-js: 0.11.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + draft-js: 0.11.7(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) immutable: 5.1.5 html-url-attributes@3.0.1: {} @@ -39359,12 +39434,12 @@ snapshots: react: 18.3.1 scheduler: 0.23.2 - react-draft-wysiwyg@1.15.0(draft-js@0.11.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(immutable@5.1.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + react-draft-wysiwyg@1.15.0(draft-js@0.11.7(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(immutable@5.1.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: classnames: 2.5.1 - draft-js: 0.11.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1) - draftjs-utils: 0.10.2(draft-js@0.11.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(immutable@5.1.5) - html-to-draftjs: 1.5.0(draft-js@0.11.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(immutable@5.1.5) + draft-js: 0.11.7(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + draftjs-utils: 0.10.2(draft-js@0.11.7(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(immutable@5.1.5) + html-to-draftjs: 1.5.0(draft-js@0.11.7(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(immutable@5.1.5) immutable: 5.1.5 linkify-it: 2.2.0 prop-types: 15.8.1 @@ -42360,6 +42435,25 @@ snapshots: tsx: 4.21.0 yaml: 2.8.2 + vite@7.3.1(@types/node@20.19.26)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2): + dependencies: + esbuild: 0.27.4 + fdir: 6.5.0(picomatch@4.0.3) + picomatch: 4.0.3 + postcss: 8.5.8 + rollup: 4.59.0 + tinyglobby: 0.2.15 + optionalDependencies: + '@types/node': 20.19.26 + fsevents: 2.3.3 + jiti: 2.6.1 + lightningcss: 1.31.1 + sass: 1.98.0 + terser: 5.46.0 + tsx: 4.21.0 + yaml: 2.8.2 + optional: true + vite@7.3.1(@types/node@22.19.15)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2): dependencies: esbuild: 0.27.4 @@ -42547,7 +42641,7 @@ snapshots: optionalDependencies: '@types/debug': 4.1.12 '@types/node': 20.19.26 - '@vitest/browser': 3.2.4(bufferutil@4.1.0)(playwright@1.58.2)(vite@6.4.1(@types/node@20.19.26)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))(vitest@3.2.4) + '@vitest/browser': 3.2.4(bufferutil@4.1.0)(playwright@1.58.2)(vite@7.3.1(@types/node@20.19.26)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))(vitest@3.2.4) jsdom: 19.0.0(bufferutil@4.1.0) transitivePeerDependencies: - jiti @@ -42591,7 +42685,7 @@ snapshots: optionalDependencies: '@types/debug': 4.1.12 '@types/node': 22.19.15 - '@vitest/browser': 3.2.4(bufferutil@4.1.0)(playwright@1.58.2)(vite@7.3.1(@types/node@22.19.15)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))(vitest@3.2.4) + '@vitest/browser': 3.2.4(bufferutil@4.1.0)(playwright@1.58.2)(vite@6.4.1(@types/node@22.19.15)(jiti@2.6.1)(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))(vitest@3.2.4) jsdom: 19.0.0(bufferutil@4.1.0) transitivePeerDependencies: - jiti diff --git a/services/forgejo-code-sync/.gitignore b/services/forgejo-code-sync/.gitignore new file mode 100644 index 000000000..ce4da2609 --- /dev/null +++ b/services/forgejo-code-sync/.gitignore @@ -0,0 +1,3 @@ +.queue/ +.queue-snapshots/ +.repo-envelopes/ diff --git a/services/forgejo-code-sync/README.md b/services/forgejo-code-sync/README.md new file mode 100644 index 000000000..5d97f8319 --- /dev/null +++ b/services/forgejo-code-sync/README.md @@ -0,0 +1,129 @@ +# forgejo-code-sync + +Syncs commits pushed to GitW3 into the pushing author's own eVault, and keeps a full up-to-date copy of the repo in +its **owner's** own eVault - two independent sync paths off the same webhook, see [What gets synced](#what-gets-synced). + +**Design:** [docs/superpowers/specs/2026-08-14-forgejo-code-sync-design.md](../../docs/superpowers/specs/2026-08-14-forgejo-code-sync-design.md) +**Plan:** [docs/superpowers/plans/2026-08-14-forgejo-code-sync-plan.md](../../docs/superpowers/plans/2026-08-14-forgejo-code-sync-plan.md) + +## The two contracts + +**Forgejo side** is a system webhook - configured once, instance-wide, no per-repo setup - delivering every `push` +event. `pusher.login` identifies the authenticated account that ran `git push`; each commit's own `author`/`committer` +fields are free-text `git config`, never validated against any Forgejo account, and this service never resolves +identity from them. The webhook's `X-Forgejo-Signature` header is a raw, unprefixed HMAC-SHA256 hex digest over the +exact request bytes - not the GitHub-style `sha256=`-prefixed `X-Hub-Signature-256` Forgejo sends alongside it for +compatibility. + +**eVault side** is the same certify-then-per-eName-GraphQL pattern used by +[`platforms/calendar/api/src/services/EVaultService.ts`](../../platforms/calendar/api/src/services/EVaultService.ts): +this service certifies itself with the Registry once, then writes into *the pusher's own* eVault by presenting their +eName as `X-ENAME` on each write. + +The identity link between the two is the [w3ds-oidc-bridge](../w3ds-oidc-bridge/README.md): once someone signs into +GitW3 through it, GitW3's `login_name` for that account holds their full eName (`@` included). This service resolves +`pusher.login -> login_name -> eName` via `GET /api/v1/users/{username}`, which only returns `login_name` to an +admin-authenticated caller - see [Configuration](#configuration) for what that requires of `FORGEJO_ADMIN_TOKEN`. + +## What gets synced + +One MetaEnvelope per commit (`services/ontology/schemas/codeCommit.json`), written into the pusher's eVault with an +`acl` that mirrors the source repository's visibility at push time: `["*"]` for a public repo, owner-only for a +private one. A commit's diff is inlined when it fits under `FORGEJO_SYNC_DIFF_MAX_BYTES`, and replaced with a +`diffUrl` pointer back to GitW3 otherwise - or on any fetch failure, so an oversized or momentarily unreachable diff +never blocks the commit's own metadata from being synced. + +A push from an account with no linked eVault (`login_name` doesn't start with `@`) is skipped silently - the ordinary +case for most GitW3 accounts, not a failure. + +**A second, independent path off the same webhook** writes the repo's complete current state - every file and +folder, via GitW3's archive endpoint, uploaded to S3 - into the **owner's** own eVault +(`services/ontology/schemas/repoSnapshot.json`), replaced in place on every push rather than accumulating one +envelope per push. `repository.owner.login`, not `pusher.login`, is resolved the same way (`IdentityResolver` is +generic over any Forgejo username); an org-owned repo or an owner with no linked eVault skips the same way an +unlinked pusher does. See the spec's [Repo-owner full snapshot](../../docs/superpowers/specs/2026-08-14-forgejo-code-sync-design.md#repo-owner-full-snapshot-added-2026-08-15) +section for the full design and live-verification detail. + +## Delivery reliability + +Forgejo has no automatic retry or redelivery of failed webhook deliveries at all - confirmed against GitW3's own +`services/webhook/deliver.go`. So this service owns its own reliability: every commit is durably queued to disk +(`.queue/` locally - see [Deployment](#deployment)) before the webhook handler responds, and a failed sync is retried +with exponential backoff rather than dropped. A task that exhausts its retry budget is left on disk in an `exhausted` +status - logged distinctly from an ordinary skip - rather than silently removed, since it needs a human. + +The repo-owner snapshot sync has its own independent queue (`.queue-snapshots/`), same reliability discipline, same +skip/retry/exhausted semantics - a slow or down eVault delays the owner's snapshot the same way it delays a pusher's +commit, and neither queue's failure affects the other's. + +## Configuration + +Read from the repository root `.env`, same `required()`-throws-at-startup pattern as the bridge's own `config.ts`. + +| Variable | Default | Note | +|---|---|---| +| `FORGEJO_SYNC_PUBLIC_URL` | - | this service's own base URL, used for Registry platform certification and to build the webhook URL registered on GitW3 | +| `FORGEJO_SYNC_PORT` | `4300` | | +| `FORGEJO_WEBHOOK_SECRET` | - | HMAC secret configured on the Forgejo system webhook | +| `FORGEJO_API_URL` | - | GitW3's base URL | +| `FORGEJO_ADMIN_TOKEN` | - | PAT on a **dedicated site-admin service account**, scopes `read:user,read:repository` - see below | +| `FORGEJO_SYNC_DIFF_MAX_BYTES` | `131072` | inline cap before falling back to `diffUrl` | +| `PUBLIC_REGISTRY_URL` | - | already in the root `.env` | +| `PUBLIC_EVAULT_SERVER_URI` | - | already in the root `.env` | + +### Why the admin token has to be this big + +`GET /api/v1/users/{username}` only returns `login_name` when the caller is the account itself or a site admin - +scope alone doesn't gate it, confirmed against `services/convert/user.go`'s `toUser`. Fetching a private repo's diff +separately requires `read:repository`. So `FORGEJO_ADMIN_TOKEN` has to be a PAT belonging to an actual site-admin +account, not merely one carrying those scopes - and because that token can read every account's `login_name` and +every repo's content, not just what this service needs at a given moment, it should be a service account created for +this purpose alone, never a shared human admin's personal token. See the spec's Trust model for the full reasoning, +including the accepted limitation this implies for the ACL decision below. + +## Running locally + +```bash +pnpm --filter forgejo-code-sync dev +``` + +Then register the webhook against a local GitW3 instance (idempotent - safe to re-run): + +```bash +pnpm --filter forgejo-code-sync register-webhook +``` + +`GET /healthz` returns `200` once the service is up. Pushing to a repo whose owner has signed into that GitW3 through +the bridge should produce a `codeCommit` MetaEnvelope in their eVault within one drain cycle (5s). + +## Testing + +```bash +pnpm --filter forgejo-code-sync test +``` + +Everything through identity resolution, signature verification, ACL derivation, the retry queue, and the drain loop +is covered without a live GitW3, eVault, or Registry - every external call is stubbed. What isn't covered by +automated tests: a real end-to-end run against a live GitW3 + bridge + eVault, which needs the same manual walkthrough +the bridge's own README describes for testing without a phone, extended by pushing a commit as the final step - see +the plan's Phase 6. + +**Live verification, done three times, not just planned.** See the spec's Testing section for all three passes: the +first live push (public/private repo, unlinked-account skip, single-webhook regression, plus two GitW3 +webhook-provisioning traps found only by testing); the 2026-08-15 follow-up that closed the one gap the first pass +left open - a real differential proof that a private-repo diff's S3 object is anonymously unreachable while a public +one isn't, plus a fourth webhook trap (`PATCH /admin/hooks/{id}` silently ignores a secret rotation - fixed in +`scripts/register-webhook.ts`, which now deletes and recreates instead of patching); and the same day's repo-owner +snapshot pass - the archive endpoint found and live-verified before any code was written against it, a real two-push +sequence proving update-in-place (same envelope id, S3 object content changed), the same S3 ACL differential proof +applied to a repo archive, and the org/unlinked-owner skip path exercised live. + +## Deployment + +Inherits the bridge's own unresolved blocker: **no service deployment manifest exists in this repository for a +production or staging host.** See the spec's Deployment section. `docker/Dockerfile.forgejo-code-sync` follows the +`docker/Dockerfile.` convention once a host is known. + +Where the retry queue persists in a real deployment is part of that same open question - the local default is a +`.queue/` directory next to the package (gitignored), which is enough for development but not a production storage +decision. diff --git a/services/forgejo-code-sync/package.json b/services/forgejo-code-sync/package.json new file mode 100644 index 000000000..2c6edffa0 --- /dev/null +++ b/services/forgejo-code-sync/package.json @@ -0,0 +1,33 @@ +{ + "name": "forgejo-code-sync", + "version": "0.1.0", + "description": "Syncs commits pushed to GitW3 into the pushing author's eVault, resolved via the identity the w3ds-oidc-bridge links on sign-in", + "type": "module", + "private": true, + "main": "./dist/index.js", + "scripts": { + "dev": "tsx watch src/index.ts", + "build": "tsc -p tsconfig.build.json", + "start": "node dist/index.js", + "register-webhook": "tsx scripts/register-webhook.ts", + "test": "vitest run", + "test:watch": "vitest", + "check": "npx @biomejs/biome check ./src && tsc --noEmit", + "check-format": "npx @biomejs/biome format ./src", + "check-lint": "npx @biomejs/biome lint ./src", + "check-types": "tsc --noEmit" + }, + "dependencies": { + "@aws-sdk/client-s3": "^3.700.0", + "dotenv": "^16.4.5", + "express": "^4.18.2", + "graphql-request": "^6.1.0" + }, + "devDependencies": { + "@types/express": "^4.17.21", + "@types/node": "^20.11.24", + "tsx": "^4.7.1", + "typescript": "^5.3.3", + "vitest": "^3.0.9" + } +} diff --git a/services/forgejo-code-sync/scripts/register-webhook.ts b/services/forgejo-code-sync/scripts/register-webhook.ts new file mode 100644 index 000000000..1f425c903 --- /dev/null +++ b/services/forgejo-code-sync/scripts/register-webhook.ts @@ -0,0 +1,180 @@ +#!/usr/bin/env node +import path from "node:path"; +import { fileURLToPath } from "node:url"; +/** + * Registers - or updates - the system webhook GitW3 sends every push to. + * + * Unlike the bridge's authentication source (registered via the `gitea` CLI + * against the shared data volume, see docker/gitw3-register-auth-source.sh), + * there is no CLI subcommand for webhooks at all - checked directly against + * GitW3's `cmd/` source, nothing matches. This is why the mechanism here is + * the Admin REST API (`POST /api/v1/admin/hooks`, + * routers/api/v1/admin/hooks.go's `CreateHook`) instead: a "system webhook", + * scoped instance-wide, exactly what Site Administration's UI would create by + * hand. Idempotent, so it can run on every deploy: it updates the existing + * hook when one already points at this service's /webhook URL, and creates + * one otherwise - by deleting and recreating rather than PATCHing when one + * already exists, since PATCH silently ignores a changed secret (see the + * comment above the delete-then-recreate call below). + * + * Uses FORGEJO_PROVISIONING_TOKEN, not FORGEJO_ADMIN_TOKEN - confirmed live, + * not just from source: /admin/hooks requires AccessTokenScopeCategoryAdmin + * (routers/api/v1/api.go:1415, "write:admin" for a create/update), which + * FORGEJO_ADMIN_TOKEN deliberately does NOT carry - it only needs + * read:user,read:repository for what the running service actually does + * forever. Provisioning a webhook is a one-time operator action; the + * always-on service should not sit on a credential that could also rewrite + * every system webhook, when it never needs to. Falls back to + * FORGEJO_ADMIN_TOKEN if the narrower token isn't set, so a single-token + * setup still works - just with a wider blast radius than necessary. + * + * Run from the repository root: + * pnpm --filter forgejo-code-sync register-webhook + */ +import { config as loadEnv } from "dotenv"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +loadEnv({ path: path.resolve(here, "../../../.env") }); + +function required(name: string): string { + const value = process.env[name]?.trim(); + if (!value) { + throw new Error(`Missing required environment variable: ${name}`); + } + return value; +} + +interface Hook { + id: number; + url: string; + active: boolean; +} + +/** + * Pages through every system webhook. `GET /admin/hooks` is paginated + * (`page`/`limit` query params, routers/api/v1/admin/hooks.go's `ListHooks`) + * - a naive single-page fetch would miss an existing hook past the default + * page size and register a duplicate instead of updating it. + */ +async function listAllHooks( + forgejoApiUrl: string, + adminToken: string, +): Promise { + const hooks: Hook[] = []; + for (let page = 1; ; page++) { + const res = await fetch( + `${forgejoApiUrl}/api/v1/admin/hooks?page=${page}&limit=50`, + { headers: { Authorization: `token ${adminToken}` } }, + ); + if (!res.ok) { + throw new Error(`GET /admin/hooks failed: HTTP ${res.status}`); + } + const page_ = (await res.json()) as Hook[]; + if (page_.length === 0) break; + hooks.push(...page_); + } + return hooks; +} + +async function main(): Promise { + const forgejoApiUrl = required("FORGEJO_API_URL").replace(/\/+$/, ""); + const adminToken = + process.env.FORGEJO_PROVISIONING_TOKEN?.trim() || + required("FORGEJO_ADMIN_TOKEN"); + const webhookSecret = required("FORGEJO_WEBHOOK_SECRET"); + const publicUrl = required("FORGEJO_SYNC_PUBLIC_URL").replace(/\/+$/, ""); + const webhookUrl = `${publicUrl}/webhook`; + + const authHeaders = { + Authorization: `token ${adminToken}`, + "Content-Type": "application/json", + }; + + // Two traps in the same endpoint, both confirmed live against a running + // GitW3, not just from source - neither produces an error, which is what + // makes them dangerous. + // + // 1. `active` defaults to false (CreateHookOption.Active bool, zero + // value) if omitted. Omitting it still returns 201 with a real hook id + // - Site Administration shows it, GET /admin/hooks lists it - but it + // silently never delivers a single push. + // 2. POST /admin/hooks creates a "default" webhook, NOT a system one, + // unless config.is_system_webhook is the *string* "true" + // (routers/api/v1/utils/hook.go's addHook: `isSystemWebhook` only + // becomes true when `form.Config["is_system_webhook"]` parses truthy; + // the field isn't in CreateHookOption's documented shape at all, it's + // read out of the free-form `config` map). A default webhook is only + // copied into repos created *after* it's added - it does not apply + // retroactively to existing repos, and GetSystemWebhooks + // (models/webhook/webhook_system.go) filters `is_system_webhook=true`, + // so it is invisible to GET /admin/hooks too. Confirmed by reproducing + // it: without this field, the create call returns 201, but the hook + // never appears in a follow-up GET /admin/hooks and would silently + // miss every already-existing repo - exactly the "sync every push" + // requirement this service exists to satisfy. + // + // Both are passed explicitly, every time, on both create and update. + const body = { + type: "forgejo", + config: { + url: webhookUrl, + content_type: "json", + secret: webhookSecret, + is_system_webhook: "true", + }, + events: ["push"], + active: true, + }; + + const hooks = await listAllHooks(forgejoApiUrl, adminToken); + const existing = hooks.find((h) => h.url === webhookUrl); + + // A fourth trap, found only by testing a secret rotation live, not by + // re-reading source harder: `PATCH /admin/hooks/{id}` (editHook, + // routers/api/v1/utils/hook.go) updates `url`, `content_type`, events, + // branch_filter and the authorization header from `config` - but never + // reads `config["secret"]` at all. Reproduced directly: PATCHing an + // existing hook with a new FORGEJO_WEBHOOK_SECRET returns 200, and + // GET /admin/hooks shows nothing wrong (secret is never echoed back by + // any Forgejo API, by design) - but the hook keeps signing with + // whatever secret it was *created* with, forever. The next real push's + // delivery comes back 401 "invalid signature" against the new secret, + // silently, exactly like the `active`/`is_system_webhook` traps above. + // So a secret rotation is not a PATCH - it's delete-then-recreate, + // since only `addHook` (the POST path) reads `config["secret"]`. + if (existing) { + console.log( + `secret rotation: deleting + recreating system webhook (id ${existing.id}) -> ${webhookUrl} ` + + "(PATCH /admin/hooks/{id} silently ignores config.secret - see comment above)", + ); + const del = await fetch( + `${forgejoApiUrl}/api/v1/admin/hooks/${existing.id}`, + { method: "DELETE", headers: authHeaders }, + ); + if (!del.ok && del.status !== 404) { + throw new Error( + `DELETE /admin/hooks/${existing.id} failed: HTTP ${del.status}`, + ); + } + } else { + console.log(`creating system webhook -> ${webhookUrl}`); + } + + const res = await fetch(`${forgejoApiUrl}/api/v1/admin/hooks`, { + method: "POST", + headers: authHeaders, + body: JSON.stringify(body), + }); + if (!res.ok) { + throw new Error(`POST /admin/hooks failed: HTTP ${res.status}`); + } + + console.log( + "done - verify Active is on in Site Administration -> Webhooks, not just that the row exists.", + ); +} + +main().catch((error: unknown) => { + console.error(error); + process.exit(1); +}); diff --git a/services/forgejo-code-sync/src/app.ts b/services/forgejo-code-sync/src/app.ts new file mode 100644 index 000000000..c89036812 --- /dev/null +++ b/services/forgejo-code-sync/src/app.ts @@ -0,0 +1,36 @@ +import express, { type Express } from "express"; +import type { Queue } from "./queue.js"; +import type { CommitSyncTask, RepoSnapshotTask } from "./task.js"; +import { createPushHandlers } from "./webhook/push.js"; + +export interface AppDeps { + queue: Queue; + snapshotQueue: Queue; + webhookSecret: string; +} + +export function createApp(deps: AppDeps): Express { + const app = express(); + + // Behind TLS termination in staging and production, same as the bridge. + app.set("trust proxy", true); + app.disable("x-powered-by"); + + app.get("/healthz", (_req, res) => { + res.json({ ok: true }); + }); + + // Route-scoped raw-body parsing lives in createPushHandlers itself - see + // webhook/push.ts for why a global express.json() would be the wrong tool + // here. + app.post( + "/webhook", + ...createPushHandlers({ + commitQueue: deps.queue, + snapshotQueue: deps.snapshotQueue, + webhookSecret: deps.webhookSecret, + }), + ); + + return app; +} diff --git a/services/forgejo-code-sync/src/config.test.ts b/services/forgejo-code-sync/src/config.test.ts new file mode 100644 index 000000000..39313c959 --- /dev/null +++ b/services/forgejo-code-sync/src/config.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from "vitest"; +import { ConfigError, loadConfig } from "./config.js"; + +const complete: NodeJS.ProcessEnv = { + FORGEJO_SYNC_PUBLIC_URL: "https://forgejo-sync.example.org", + FORGEJO_WEBHOOK_SECRET: "secret", + FORGEJO_API_URL: "https://git.example.org", + FORGEJO_ADMIN_TOKEN: "token", + PUBLIC_REGISTRY_URL: "https://registry.example.org", + PUBLIC_EVAULT_SERVER_URI: "https://evault.example.org", + DO_SPACES_ENDPOINT: "https://nyc3.digitaloceanspaces.com", + DO_SPACES_REGION: "nyc3", + DO_SPACES_KEY: "spaces-key", + DO_SPACES_SECRET: "spaces-secret", + DO_SPACES_BUCKET: "spaces-bucket", +}; + +const env = (overrides: NodeJS.ProcessEnv = {}) => ({ + ...complete, + ...overrides, +}); + +describe("loadConfig", () => { + it("accepts a complete environment", () => { + const config = loadConfig(env()); + expect(config.webhookSecret).toBe("secret"); + expect(config.port).toBe(4300); + expect(config.s3.bucket).toBe("spaces-bucket"); + expect(config.s3.cdnUrl).toBeUndefined(); + }); + + it("accepts an optional DO_SPACES_CDN_URL", () => { + const config = loadConfig( + env({ DO_SPACES_CDN_URL: "https://cdn.example.org" }), + ); + expect(config.s3.cdnUrl).toBe("https://cdn.example.org"); + }); + + describe("required keys", () => { + const keys = [ + "FORGEJO_SYNC_PUBLIC_URL", + "FORGEJO_WEBHOOK_SECRET", + "FORGEJO_API_URL", + "FORGEJO_ADMIN_TOKEN", + "PUBLIC_REGISTRY_URL", + "PUBLIC_EVAULT_SERVER_URI", + "DO_SPACES_ENDPOINT", + "DO_SPACES_REGION", + "DO_SPACES_KEY", + "DO_SPACES_SECRET", + "DO_SPACES_BUCKET", + ]; + + it.each(keys)("throws naming %s when it is missing", (key) => { + const incomplete = env(); + delete incomplete[key]; + // Naming the key matters: this error is the whole diagnostic a + // deployer gets, matching the bridge's own config.ts. + expect(() => loadConfig(incomplete)).toThrowError(new RegExp(key)); + }); + + it.each(keys)("treats %s set to whitespace as missing", (key) => { + expect(() => loadConfig(env({ [key]: " " }))).toThrowError( + ConfigError, + ); + }); + }); + + describe("URL normalisation", () => { + it("strips a trailing slash from FORGEJO_SYNC_PUBLIC_URL", () => { + expect( + loadConfig( + env({ FORGEJO_SYNC_PUBLIC_URL: "https://b.example.org/" }), + ).publicUrl, + ).toBe("https://b.example.org"); + }); + + it("strips a trailing slash from FORGEJO_API_URL", () => { + expect( + loadConfig(env({ FORGEJO_API_URL: "https://git.example.org/" })) + .forgejoApiUrl, + ).toBe("https://git.example.org"); + }); + + it("strips a trailing slash from PUBLIC_EVAULT_SERVER_URI", () => { + expect( + loadConfig( + env({ + PUBLIC_EVAULT_SERVER_URI: "https://evault.example.org/", + }), + ).evaultServerUri, + ).toBe("https://evault.example.org"); + }); + }); + + describe("the port", () => { + it("parses a value", () => { + expect(loadConfig(env({ FORGEJO_SYNC_PORT: "5000" })).port).toBe( + 5000, + ); + }); + + it.each(["nope", "0", "70000", "4300.5"])("rejects %s", (value) => { + expect(() => + loadConfig(env({ FORGEJO_SYNC_PORT: value })), + ).toThrowError(ConfigError); + }); + }); +}); diff --git a/services/forgejo-code-sync/src/config.ts b/services/forgejo-code-sync/src/config.ts new file mode 100644 index 000000000..33d721066 --- /dev/null +++ b/services/forgejo-code-sync/src/config.ts @@ -0,0 +1,106 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { config as loadEnv } from "dotenv"; + +export interface SyncConfig { + /** This service's own base URL, used for Registry platform certification. */ + publicUrl: string; + port: number; + /** HMAC secret configured on the Forgejo system webhook. */ + webhookSecret: string; + /** GitW3's base URL, for the admin Users API call and diff fetching. */ + forgejoApiUrl: string; + /** + * PAT on a dedicated site-admin service account, scopes read:user,read:repository. + * read:user alone is not enough - login_name is only returned to a caller whose + * account has IsAdmin=true, regardless of token scope. See the spec's Trust model. + */ + forgejoAdminToken: string; + registryUrl: string; + evaultServerUri: string; + /** + * DigitalOcean Spaces (S3-compatible) - the same bucket evault-core's own + * StorageService.ts uses. Diffs are uploaded here directly rather than + * through evault-core's `uploadFile` GraphQL mutation, which caps at + * 250MB; S3 itself has no such ceiling. + */ + s3: { + endpoint: string; + region: string; + accessKeyId: string; + secretAccessKey: string; + bucket: string; + cdnUrl?: string; + }; +} + +export class ConfigError extends Error {} + +function required(env: NodeJS.ProcessEnv, name: string): string { + const value = env[name]?.trim(); + if (!value) { + throw new ConfigError(`Missing required environment variable: ${name}`); + } + return value; +} + +function optional( + env: NodeJS.ProcessEnv, + name: string, + fallback: string, +): string { + const value = env[name]?.trim(); + return value ? value : fallback; +} + +/** + * Builds the configuration from an environment. Pure: it reads nothing but the + * map it is handed, so tests do not have to mutate `process.env`. + * + * Throws rather than degrading, matching the bridge's own config.ts - a service + * that starts with a missing admin token fails later, at a point where the + * symptom (an unresolved eName on every push) no longer points at the cause. + */ +export function loadConfig(env: NodeJS.ProcessEnv = process.env): SyncConfig { + const port = Number(optional(env, "FORGEJO_SYNC_PORT", "4300")); + if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new ConfigError( + `FORGEJO_SYNC_PORT must be an integer between 1 and 65535: ${env.FORGEJO_SYNC_PORT}`, + ); + } + + return { + publicUrl: required(env, "FORGEJO_SYNC_PUBLIC_URL").replace(/\/+$/, ""), + port, + webhookSecret: required(env, "FORGEJO_WEBHOOK_SECRET"), + forgejoApiUrl: required(env, "FORGEJO_API_URL").replace(/\/+$/, ""), + forgejoAdminToken: required(env, "FORGEJO_ADMIN_TOKEN"), + registryUrl: required(env, "PUBLIC_REGISTRY_URL"), + evaultServerUri: required(env, "PUBLIC_EVAULT_SERVER_URI").replace( + /\/+$/, + "", + ), + s3: { + endpoint: required(env, "DO_SPACES_ENDPOINT"), + region: required(env, "DO_SPACES_REGION"), + accessKeyId: required(env, "DO_SPACES_KEY"), + secretAccessKey: required(env, "DO_SPACES_SECRET"), + bucket: required(env, "DO_SPACES_BUCKET"), + cdnUrl: env.DO_SPACES_CDN_URL?.trim() || undefined, + }, + }; +} + +let cached: SyncConfig | undefined; + +/** Memoised singleton for the running service. Loads the repository root `.env`. */ +export function getConfig(): SyncConfig { + if (!cached) { + const here = path.dirname(fileURLToPath(import.meta.url)); + // src/ during development, dist/ once built - both sit one level under + // the package, so the same relative path reaches the repository root. + loadEnv({ path: path.resolve(here, "../../../.env") }); + cached = loadConfig(); + } + return cached; +} diff --git a/services/forgejo-code-sync/src/content/archive.test.ts b/services/forgejo-code-sync/src/content/archive.test.ts new file mode 100644 index 000000000..2dced924f --- /dev/null +++ b/services/forgejo-code-sync/src/content/archive.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, it, vi } from "vitest"; +import type { S3Storage } from "../storage/s3.js"; +import type { RepoSnapshotTask } from "../task.js"; +import { createArchiveFetcher } from "./archive.js"; + +const task: RepoSnapshotTask = { + repoFullName: "alice/repo", + repoPrivate: false, + ref: "refs/heads/main", + ownerLogin: "alice", + headCommitId: "abc123", +}; + +function fakeStorage( + uploadRepoArchive = vi + .fn() + .mockResolvedValue("https://s3.example.org/repos/alice/repo.zip"), +) { + return { uploadRepoArchive } as unknown as S3Storage; +} + +describe("createArchiveFetcher", () => { + it("fetches from the API router's archive/{sha}.zip route, using the push's headCommitId as the ref", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(new Response(new Uint8Array([1, 2, 3]))); + const storage = fakeStorage(); + const fetchArchive = createArchiveFetcher({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + storage, + fetchImpl, + }); + + await fetchArchive(task); + + expect(fetchImpl).toHaveBeenCalledWith( + "https://git.example.org/api/v1/repos/alice/repo/archive/abc123.zip", + { headers: { Authorization: "token admin-token" } }, + ); + }); + + it("uploads the fetched archive bytes to S3 and returns the resulting URL", async () => { + const archiveBytes = new Uint8Array([80, 75, 3, 4]); // zip magic bytes + const fetchImpl = vi.fn().mockResolvedValue(new Response(archiveBytes)); + const uploadRepoArchive = vi + .fn() + .mockResolvedValue("https://s3.example.org/repos/alice/repo.zip"); + const fetchArchive = createArchiveFetcher({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + storage: fakeStorage(uploadRepoArchive), + fetchImpl, + }); + + const url = await fetchArchive(task); + + expect(url).toBe("https://s3.example.org/repos/alice/repo.zip"); + expect(uploadRepoArchive).toHaveBeenCalledWith( + "alice/repo", + expect.any(Buffer), + true, // !task.repoPrivate + ); + }); + + it("uploads with isPublic=false for a private repo", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(new Response(new Uint8Array([1]))); + const uploadRepoArchive = vi + .fn() + .mockResolvedValue("https://s3.example.org/x"); + const fetchArchive = createArchiveFetcher({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + storage: fakeStorage(uploadRepoArchive), + fetchImpl, + }); + + await fetchArchive({ ...task, repoPrivate: true }); + + expect(uploadRepoArchive).toHaveBeenCalledWith( + "alice/repo", + expect.any(Buffer), + false, + ); + }); + + it("throws on a non-2xx response, rather than degrading to a fallback", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(new Response(null, { status: 404 })); + const fetchArchive = createArchiveFetcher({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + storage: fakeStorage(), + fetchImpl, + }); + + await expect(fetchArchive(task)).rejects.toThrow(/404/); + }); + + it("throws on a network failure, rather than degrading to a fallback", async () => { + const fetchImpl = vi.fn().mockRejectedValue(new Error("ECONNREFUSED")); + const fetchArchive = createArchiveFetcher({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + storage: fakeStorage(), + fetchImpl, + }); + + await expect(fetchArchive(task)).rejects.toThrow("ECONNREFUSED"); + }); + + it("throws when the S3 upload itself fails", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(new Response(new Uint8Array([1]))); + const uploadRepoArchive = vi + .fn() + .mockRejectedValue(new Error("bucket unreachable")); + const fetchArchive = createArchiveFetcher({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + storage: fakeStorage(uploadRepoArchive), + fetchImpl, + }); + + await expect(fetchArchive(task)).rejects.toThrow("bucket unreachable"); + }); +}); diff --git a/services/forgejo-code-sync/src/content/archive.ts b/services/forgejo-code-sync/src/content/archive.ts new file mode 100644 index 000000000..e0b07307d --- /dev/null +++ b/services/forgejo-code-sync/src/content/archive.ts @@ -0,0 +1,68 @@ +import type { S3Storage } from "../storage/s3.js"; +import type { RepoSnapshotTask } from "../task.js"; + +export interface ArchiveFetcherOptions { + forgejoApiUrl: string; + /** PAT on a dedicated site-admin service account - needs read:repository, same token content/diff.ts uses. */ + adminToken: string; + storage: S3Storage; + fetchImpl?: typeof fetch; +} + +/** + * Fetches a full repo archive from GitW3 at the push's final ref state and + * uploads it to S3, returning the resulting URL - never inlined, same + * reasoning as content/diff.ts, and doubly true here since a whole repo is + * routinely far larger than a single diff. + * + * GitW3-verified live, not assumed from source alone, per the same discipline + * that caught the web-router `.diff` route's private-repo auth bug: curled + * `GET /api/v1/repos/{owner}/{repo}/archive/{sha}.zip` directly against a + * live private test repo. `Authorization: token ` -> `200` with + * a real zip archive containing the repo's actual files; the identical + * request with no `Authorization` header -> `404` (Forgejo's usual + * anonymous-request-on-a-private-repo response, not `403`, so as not to leak + * that the repo exists). Confirmed with a real commit sha as the ref, not + * just a branch name, since that's what the once-per-push snapshot task + * always has (`RepoSnapshotTask.headCommitId`, the push payload's own + * `after`). + * + * Source confirms this is on the same PAT-aware auth chain as the working + * `.diff` route, not the broken one: `routers/api/v1/api.go`'s + * `m.Get("/archive/*", reqRepoReader(unit.TypeCode), repo.GetArchive)`, + * mounted under `/api/v1/repos/{username}/{reponame}` - the API router, never + * the web router's own separate `/archive` group in `routers/web/web.go`, + * which is not used here at all. + * + * Throws on any failure - a fetch error, a non-2xx response, an S3 upload + * failure - same discipline as content/diff.ts: there is no degraded + * fallback to fall back to, so a failure here means the whole snapshot task + * retries via the snapshot queue's own backoff. + */ +export function createArchiveFetcher(options: ArchiveFetcherOptions) { + const fetchImpl = options.fetchImpl ?? fetch; + + return async function fetchArchive( + task: RepoSnapshotTask, + ): Promise { + const url = `${options.forgejoApiUrl}/api/v1/repos/${task.repoFullName}/archive/${task.headCommitId}.zip`; + + const res = await fetchImpl(url, { + headers: { Authorization: `token ${options.adminToken}` }, + }); + + if (!res.ok) { + throw new Error( + `fetching archive for ${task.repoFullName}@${task.headCommitId} failed: HTTP ${res.status}`, + ); + } + + const archiveBytes = Buffer.from(await res.arrayBuffer()); + + return options.storage.uploadRepoArchive( + task.repoFullName, + archiveBytes, + !task.repoPrivate, + ); + }; +} diff --git a/services/forgejo-code-sync/src/content/diff.test.ts b/services/forgejo-code-sync/src/content/diff.test.ts new file mode 100644 index 000000000..433e6dd0f --- /dev/null +++ b/services/forgejo-code-sync/src/content/diff.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it, vi } from "vitest"; +import type { S3Storage } from "../storage/s3.js"; +import type { CommitSyncTask } from "../task.js"; +import { createDiffFetcher } from "./diff.js"; + +const task: CommitSyncTask = { + commitId: "abc123", + repoFullName: "alice/repo", + repoPrivate: false, + ref: "refs/heads/main", + pusherLogin: "alice", + message: "a commit", + committedAt: "2026-08-14T10:00:00Z", + added: [], + removed: [], + modified: [], + commitUrl: "https://git.example.org/alice/repo/commit/abc123", + compareUrl: "https://git.example.org/alice/repo/compare/x...y", +}; + +function fakeStorage( + uploadDiff = vi + .fn() + .mockResolvedValue( + "https://s3.example.org/diffs/alice/repo/abc123.diff", + ), +) { + return { uploadDiff } as unknown as S3Storage; +} + +describe("createDiffFetcher", () => { + it("fetches from the API router's git/commits/{sha}.diff route, not the web router's commit/{sha}.diff", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue( + new Response("diff --git a/a.ts b/a.ts", { status: 200 }), + ); + const storage = fakeStorage(); + const fetchDiff = createDiffFetcher({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + storage, + fetchImpl, + }); + + await fetchDiff(task, "@alice"); + + expect(fetchImpl).toHaveBeenCalledWith( + "https://git.example.org/api/v1/repos/alice/repo/git/commits/abc123.diff", + { headers: { Authorization: "token admin-token" } }, + ); + }); + + it("uploads the fetched diff text to S3 and returns the resulting URL", async () => { + const diffText = "diff --git a/a.ts b/a.ts\n+hello\n"; + const fetchImpl = vi + .fn() + .mockResolvedValue(new Response(diffText, { status: 200 })); + const uploadDiff = vi + .fn() + .mockResolvedValue( + "https://s3.example.org/diffs/alice/repo/abc123.diff", + ); + const fetchDiff = createDiffFetcher({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + storage: fakeStorage(uploadDiff), + fetchImpl, + }); + + const url = await fetchDiff(task, "@alice"); + + expect(url).toBe("https://s3.example.org/diffs/alice/repo/abc123.diff"); + expect(uploadDiff).toHaveBeenCalledWith( + "@alice", + "alice/repo", + "abc123", + diffText, + true, // !task.repoPrivate + ); + }); + + it("uploads with isPublic=false for a private repo", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(new Response("diff", { status: 200 })); + const uploadDiff = vi + .fn() + .mockResolvedValue("https://s3.example.org/x"); + const fetchDiff = createDiffFetcher({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + storage: fakeStorage(uploadDiff), + fetchImpl, + }); + + await fetchDiff({ ...task, repoPrivate: true }, "@alice"); + + expect(uploadDiff).toHaveBeenCalledWith( + "@alice", + "alice/repo", + "abc123", + "diff", + false, + ); + }); + + it("throws on a non-2xx response, rather than degrading to a fallback", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(new Response(null, { status: 404 })); + const fetchDiff = createDiffFetcher({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + storage: fakeStorage(), + fetchImpl, + }); + + await expect(fetchDiff(task, "@alice")).rejects.toThrow(/404/); + }); + + it("throws on a network failure, rather than degrading to a fallback", async () => { + const fetchImpl = vi.fn().mockRejectedValue(new Error("ECONNREFUSED")); + const fetchDiff = createDiffFetcher({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + storage: fakeStorage(), + fetchImpl, + }); + + await expect(fetchDiff(task, "@alice")).rejects.toThrow("ECONNREFUSED"); + }); + + it("throws when the S3 upload itself fails", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(new Response("diff", { status: 200 })); + const uploadDiff = vi + .fn() + .mockRejectedValue(new Error("bucket unreachable")); + const fetchDiff = createDiffFetcher({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + storage: fakeStorage(uploadDiff), + fetchImpl, + }); + + await expect(fetchDiff(task, "@alice")).rejects.toThrow( + "bucket unreachable", + ); + }); +}); diff --git a/services/forgejo-code-sync/src/content/diff.ts b/services/forgejo-code-sync/src/content/diff.ts new file mode 100644 index 000000000..8d5dce902 --- /dev/null +++ b/services/forgejo-code-sync/src/content/diff.ts @@ -0,0 +1,69 @@ +import type { S3Storage } from "../storage/s3.js"; +import type { CommitSyncTask } from "../task.js"; + +export interface DiffFetcherOptions { + forgejoApiUrl: string; + /** PAT on a dedicated site-admin service account - needs read:repository. */ + adminToken: string; + storage: S3Storage; + fetchImpl?: typeof fetch; +} + +/** + * Fetches a commit's diff from GitW3 and uploads it to S3, returning the + * resulting URL - the diff is never inlined into the eVault write itself. + * See docs/superpowers/specs/2026-08-14-forgejo-code-sync-design.md ("What + * gets written") for why: eVault's own server has a hard 350MB request-body + * limit, and a large blob doesn't belong inlined into a graph-database node + * property even well under that. S3 has no such ceiling. + * + * GitW3-verified: the diff is fetched from the **API router** + * (`GET /api/v1/repos/{owner}/{repo}/git/commits/{sha}.diff`, + * routers/api/v1/repo/commits.go's `DownloadCommitDiffOrPatch`), not the web + * router's `GET /{owner}/{repo}/commit/{sha}.diff`. The two look + * interchangeable but are not: the web-router route never authenticates a + * PAT for a private repo at all (confirmed empirically - Authorization: + * token, HTTP Basic, and ?token= all 404 on a private repo, while the exact + * same request succeeds the moment the repo is made public), so it always + * failed for private repos regardless of scope. The API-router route is on + * the standard PAT-aware auth chain and was confirmed working on a live + * private repo with the same token. + * + * Throws on any failure - a fetch error, a non-2xx response, an S3 upload + * failure - rather than degrading to a link back to GitW3 the way the + * previous size-capped design did. There is no longer a "couldn't get the + * diff, but here's a pointer to where you could look" fallback to degrade + * to: the point of this design is that the diff itself is what gets + * preserved, so a failure here means the whole commit-sync task retries via + * the queue's backoff, the same as an eVault write failure - see sync.ts. + */ +export function createDiffFetcher(options: DiffFetcherOptions) { + const fetchImpl = options.fetchImpl ?? fetch; + + return async function fetchDiff( + task: CommitSyncTask, + eName: string, + ): Promise { + const url = `${options.forgejoApiUrl}/api/v1/repos/${task.repoFullName}/git/commits/${task.commitId}.diff`; + + const res = await fetchImpl(url, { + headers: { Authorization: `token ${options.adminToken}` }, + }); + + if (!res.ok) { + throw new Error( + `fetching diff for ${task.repoFullName}@${task.commitId} failed: HTTP ${res.status}`, + ); + } + + const diffText = await res.text(); + + return options.storage.uploadDiff( + eName, + task.repoFullName, + task.commitId, + diffText, + !task.repoPrivate, + ); + }; +} diff --git a/services/forgejo-code-sync/src/evault/acl.test.ts b/services/forgejo-code-sync/src/evault/acl.test.ts new file mode 100644 index 000000000..2c8070972 --- /dev/null +++ b/services/forgejo-code-sync/src/evault/acl.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from "vitest"; +import { deriveAcl } from "./acl.js"; + +describe("deriveAcl", () => { + it("returns [eName] for a private repo", () => { + expect(deriveAcl(true, "@alice")).toEqual(["@alice"]); + }); + + it('returns ["*"] for a public repo', () => { + expect(deriveAcl(false, "@alice")).toEqual(["*"]); + }); + + it("does not fall back to a public ACL when eName is oddly shaped", () => { + // Not expected in practice (identity resolution already validated the + // ename before this is called), but the derivation itself must stay a + // pure function of repoIsPrivate - it must never silently widen access. + expect(deriveAcl(true, "")).toEqual([""]); + }); +}); diff --git a/services/forgejo-code-sync/src/evault/acl.ts b/services/forgejo-code-sync/src/evault/acl.ts new file mode 100644 index 000000000..762130d96 --- /dev/null +++ b/services/forgejo-code-sync/src/evault/acl.ts @@ -0,0 +1,31 @@ +/** + * The ACL to write on a synced commit's MetaEnvelope, derived from the source + * repository's visibility on GitW3. + * + * `acl: [String!]!` (infrastructure/evault-core/src/core/protocol/typedefs.ts) is + * a plain string array; `"*"` is special-cased as public everywhere it is checked. + * Every write anywhere else in this codebase uses `acl: ["*"]` with no exception, + * except one: infrastructure/evault-core/src/services/BindingDocumentService.ts + * (`acl: [normalizedSubject]` / `acl: [bindingDocument.subject]`) - a single-entry + * array holding the subject's own eName. `[eName]` below is modelled on that one + * real precedent, not invented. + * + * KNOWN LIMITATION, confirmed against evault-core's own access-control code, not + * assumed: "owner-only" here means "not public or anonymously/cross-platform- + * listable", not "cryptographically restricted to the owner". VaultAccessGuard's + * checkAccess (the resolver path for a single envelope fetched by ID) grants + * access to ANY request carrying a valid Registry-issued Bearer token from ANY + * certified platform, without consulting the envelope's acl in that branch at + * all - the acl is only actually enforced against an anonymous request (no valid + * token) or inside the bulk metaEnvelopes list query (filterEnvelopesByAccess), + * which has no such bypass. So `[eName]` reliably keeps a private-repo commit out + * of anonymous reach and out of another platform's list-query results, but does + * NOT stop a different certified platform from reading the same envelope by ID if + * it already has the ID and the right X-ENAME. That gap is evault-core's existing + * authorization model, not something this service can fix - see the spec's Trust + * model. `[eName]` is still strictly better than `["*"]`, and matches the + * codebase's only precedent for restricted data - just not an airtight guarantee. + */ +export function deriveAcl(repoIsPrivate: boolean, eName: string): string[] { + return repoIsPrivate ? [eName] : ["*"]; +} diff --git a/services/forgejo-code-sync/src/evault/client.test.ts b/services/forgejo-code-sync/src/evault/client.test.ts new file mode 100644 index 000000000..13bc993f8 --- /dev/null +++ b/services/forgejo-code-sync/src/evault/client.test.ts @@ -0,0 +1,313 @@ +import { describe, expect, it, vi } from "vitest"; +import { + CODE_COMMIT_ONTOLOGY_ID, + EVaultClient, + REPO_SNAPSHOT_ONTOLOGY_ID, +} from "./client.js"; + +function jsonResponse(status: number, body: unknown): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +const samplePayload = { + id: "abc123", + repo: "alice/repo", + ref: "refs/heads/main", + message: "a commit", + authorEName: "@alice", + committedAt: "2026-08-14T10:00:00Z", + added: ["a.ts"], + removed: [], + modified: [], + diffUrl: "https://s3.example.org/diffs/alice/repo/abc123.diff", +}; + +describe("EVaultClient.writeCommit", () => { + it("certifies once, then writes a MetaEnvelope with the right ontology and acl", async () => { + const calls: Array<{ url: string; body: unknown }> = []; + const fetchImpl = vi.fn(async (url: unknown, init?: RequestInit) => { + const urlStr = String(url); + const body = init?.body ? JSON.parse(String(init.body)) : null; + calls.push({ url: urlStr, body }); + + if (urlStr.endsWith("/platforms/certification")) { + return jsonResponse(200, { + token: "platform-token", + expiresAt: Date.now() + 3_600_000, + }); + } + if (urlStr.endsWith("/graphql")) { + return jsonResponse(200, { + data: { + createMetaEnvelope: { + metaEnvelope: { id: "envelope-1" }, + errors: [], + }, + }, + }); + } + throw new Error(`unexpected fetch: ${urlStr}`); + }); + + const client = new EVaultClient({ + registryUrl: "https://registry.example.org", + evaultServerUri: "https://evault.example.org", + publicUrl: "https://forgejo-sync.example.org", + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + const id = await client.writeCommit("@alice", samplePayload, ["*"]); + + expect(id).toBe("envelope-1"); + expect(calls[0]?.url).toBe( + "https://registry.example.org/platforms/certification", + ); + expect(calls[0]?.body).toEqual({ + platform: "https://forgejo-sync.example.org", + }); + + const graphqlCall = calls[1]; + expect(graphqlCall?.body).toMatchObject({ + variables: { + input: { + ontology: CODE_COMMIT_ONTOLOGY_ID, + payload: samplePayload, + acl: ["*"], + }, + }, + }); + }); + + it("reuses the platform token across multiple writes within its expiry", async () => { + let certifications = 0; + const fetchImpl = vi.fn(async (url: unknown) => { + const urlStr = String(url); + if (urlStr.endsWith("/platforms/certification")) { + certifications += 1; + return jsonResponse(200, { + token: "platform-token", + expiresAt: Date.now() + 3_600_000, + }); + } + return jsonResponse(200, { + data: { + createMetaEnvelope: { + metaEnvelope: { id: "envelope-x" }, + errors: [], + }, + }, + }); + }); + + const client = new EVaultClient({ + registryUrl: "https://registry.example.org", + evaultServerUri: "https://evault.example.org", + publicUrl: "https://forgejo-sync.example.org", + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + await client.writeCommit("@alice", samplePayload, ["*"]); + await client.writeCommit("@bob", samplePayload, ["@bob"]); + + expect(certifications).toBe(1); + }); + + it("throws when createMetaEnvelope returns errors", async () => { + const fetchImpl = vi.fn(async (url: unknown) => { + const urlStr = String(url); + if (urlStr.endsWith("/platforms/certification")) { + return jsonResponse(200, { + token: "platform-token", + expiresAt: Date.now() + 3_600_000, + }); + } + return jsonResponse(200, { + data: { + createMetaEnvelope: { + metaEnvelope: null, + errors: [{ message: "ontology not found" }], + }, + }, + }); + }); + + const client = new EVaultClient({ + registryUrl: "https://registry.example.org", + evaultServerUri: "https://evault.example.org", + publicUrl: "https://forgejo-sync.example.org", + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + await expect( + client.writeCommit("@alice", samplePayload, ["*"]), + ).rejects.toThrow(/ontology not found/); + }); + + it("throws when certification itself fails", async () => { + const fetchImpl = vi.fn(async () => jsonResponse(500, {})); + + const client = new EVaultClient({ + registryUrl: "https://registry.example.org", + evaultServerUri: "https://evault.example.org", + publicUrl: "https://forgejo-sync.example.org", + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + await expect( + client.writeCommit("@alice", samplePayload, ["*"]), + ).rejects.toThrow(/Failed to get platform token/); + }); +}); + +const snapshotPayload = { + repo: "alice/repo", + ref: "refs/heads/main", + headCommitId: "abc123", + ownerEName: "@alice", + snapshotUrl: "https://s3.example.org/repos/alice/repo.zip", + updatedAt: "2026-08-15T10:00:00Z", +}; + +describe("EVaultClient.writeRepoSnapshot", () => { + it("creates a new MetaEnvelope with the repoSnapshot ontology when no existing id is given", async () => { + const calls: Array<{ url: string; body: unknown }> = []; + const fetchImpl = vi.fn(async (url: unknown, init?: RequestInit) => { + const urlStr = String(url); + const body = init?.body ? JSON.parse(String(init.body)) : null; + calls.push({ url: urlStr, body }); + + if (urlStr.endsWith("/platforms/certification")) { + return jsonResponse(200, { + token: "platform-token", + expiresAt: Date.now() + 3_600_000, + }); + } + return jsonResponse(200, { + data: { + createMetaEnvelope: { + metaEnvelope: { id: "snapshot-envelope-1" }, + errors: [], + }, + }, + }); + }); + + const client = new EVaultClient({ + registryUrl: "https://registry.example.org", + evaultServerUri: "https://evault.example.org", + publicUrl: "https://forgejo-sync.example.org", + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + const id = await client.writeRepoSnapshot( + "@alice", + snapshotPayload, + ["*"], + null, + ); + + expect(id).toBe("snapshot-envelope-1"); + const graphqlCall = calls[1]; + expect(graphqlCall?.body).toMatchObject({ + query: expect.stringContaining("createMetaEnvelope"), + variables: { + input: { + ontology: REPO_SNAPSHOT_ONTOLOGY_ID, + payload: snapshotPayload, + acl: ["*"], + }, + }, + }); + }); + + it("updates the existing MetaEnvelope in place when an existing id is given, rather than creating a new one", async () => { + const calls: Array<{ url: string; body: unknown }> = []; + const fetchImpl = vi.fn(async (url: unknown, init?: RequestInit) => { + const urlStr = String(url); + const body = init?.body ? JSON.parse(String(init.body)) : null; + calls.push({ url: urlStr, body }); + + if (urlStr.endsWith("/platforms/certification")) { + return jsonResponse(200, { + token: "platform-token", + expiresAt: Date.now() + 3_600_000, + }); + } + return jsonResponse(200, { + data: { + updateMetaEnvelope: { + metaEnvelope: { id: "snapshot-envelope-1" }, + errors: [], + }, + }, + }); + }); + + const client = new EVaultClient({ + registryUrl: "https://registry.example.org", + evaultServerUri: "https://evault.example.org", + publicUrl: "https://forgejo-sync.example.org", + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + const id = await client.writeRepoSnapshot( + "@alice", + snapshotPayload, + ["*"], + "snapshot-envelope-1", + ); + + expect(id).toBe("snapshot-envelope-1"); + const graphqlCall = calls[1]; + expect(graphqlCall?.body).toMatchObject({ + query: expect.stringContaining("updateMetaEnvelope"), + variables: { + id: "snapshot-envelope-1", + input: { + ontology: REPO_SNAPSHOT_ONTOLOGY_ID, + payload: snapshotPayload, + acl: ["*"], + }, + }, + }); + }); + + it("throws when updateMetaEnvelope returns errors", async () => { + const fetchImpl = vi.fn(async (url: unknown) => { + const urlStr = String(url); + if (urlStr.endsWith("/platforms/certification")) { + return jsonResponse(200, { + token: "platform-token", + expiresAt: Date.now() + 3_600_000, + }); + } + return jsonResponse(200, { + data: { + updateMetaEnvelope: { + metaEnvelope: null, + errors: [{ message: "envelope not found" }], + }, + }, + }); + }); + + const client = new EVaultClient({ + registryUrl: "https://registry.example.org", + evaultServerUri: "https://evault.example.org", + publicUrl: "https://forgejo-sync.example.org", + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + await expect( + client.writeRepoSnapshot( + "@alice", + snapshotPayload, + ["*"], + "stale-id", + ), + ).rejects.toThrow(/envelope not found/); + }); +}); diff --git a/services/forgejo-code-sync/src/evault/client.ts b/services/forgejo-code-sync/src/evault/client.ts new file mode 100644 index 000000000..a5a8dcaff --- /dev/null +++ b/services/forgejo-code-sync/src/evault/client.ts @@ -0,0 +1,203 @@ +import { GraphQLClient } from "graphql-request"; + +/** + * Minted for this service - see services/ontology/schemas/codeCommit.json. + * No registration step beyond that file existing; the ontology service loads + * every schema in that directory at startup. + */ +export const CODE_COMMIT_ONTOLOGY_ID = "af7b8ea0-365c-414b-8dbb-5c0cdd6a46b8"; + +/** + * Minted for this service - see services/ontology/schemas/repoSnapshot.json. + * One envelope per repo, kept up to date via `writeRepoSnapshot` below rather + * than one created per push - see that method's own comment. + */ +export const REPO_SNAPSHOT_ONTOLOGY_ID = "a9b56118-ac82-4f4e-9f70-77444c1a8f34"; + +const CREATE_MUTATION = ` + mutation CreateMetaEnvelope($input: MetaEnvelopeInput!) { + createMetaEnvelope(input: $input) { + metaEnvelope { + id + } + errors { field message code } + } + } +`; + +const UPDATE_MUTATION = ` + mutation UpdateMetaEnvelope($id: ID!, $input: MetaEnvelopeInput!) { + updateMetaEnvelope(id: $id, input: $input) { + metaEnvelope { + id + } + errors { field message code } + } + } +`; + +export interface CommitEnvelopePayload { + id: string; + repo: string; + ref: string; + message: string; + authorEName: string; + committedAt: string; + added: string[]; + removed: string[]; + modified: string[]; + /** The diff's own S3 URL - see content/diff.ts. Never inlined. */ + diffUrl: string; +} + +export interface RepoSnapshotEnvelopePayload { + repo: string; + ref: string; + headCommitId: string; + ownerEName: string; + /** The repo archive's own S3 URL - see content/archive.ts. Never inlined. */ + snapshotUrl: string; + updatedAt: string; +} + +interface MetaEnvelopeMutationResult { + metaEnvelope: { id: string } | null; + errors: Array<{ message: string }> | null; +} + +interface PlatformTokenResponse { + token: string; + expiresAt?: number; +} + +export interface EVaultClientOptions { + registryUrl: string; + evaultServerUri: string; + /** This service's own public base URL, presented to the Registry for certification. */ + publicUrl: string; + fetchImpl?: typeof fetch; +} + +/** + * Certify-then-per-eName-GraphQL-client, the same shape as + * platforms/calendar/api/src/services/EVaultService.ts - not + * PlatformEVaultService.ts (used by file-manager/esigner/ecurrency/etc.), which + * provisions one eVault owned by the platform itself and is for a different + * purpose. Here the platform token authenticates *this service*; the + * X-ENAME header selects *whose* eVault a given write lands in. + */ +export class EVaultClient { + private platformToken: string | null = null; + private tokenExpiresAt = 0; + private readonly registryUrl: string; + private readonly evaultServerUri: string; + private readonly publicUrl: string; + private readonly fetchImpl: typeof fetch; + + constructor(options: EVaultClientOptions) { + this.registryUrl = options.registryUrl; + this.evaultServerUri = options.evaultServerUri; + this.publicUrl = options.publicUrl; + this.fetchImpl = options.fetchImpl ?? fetch; + } + + private async ensurePlatformToken(now = Date.now()): Promise { + if (this.platformToken && this.tokenExpiresAt > now + 5 * 60 * 1000) { + return this.platformToken; + } + + const res = await this.fetchImpl( + new URL("/platforms/certification", this.registryUrl).toString(), + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ platform: this.publicUrl }), + }, + ); + + if (!res.ok) { + throw new Error(`Failed to get platform token: HTTP ${res.status}`); + } + + const data = (await res.json()) as PlatformTokenResponse; + this.platformToken = data.token; + this.tokenExpiresAt = data.expiresAt ?? now + 3_600_000; + return this.platformToken; + } + + private async getClient(eName: string): Promise { + const token = await this.ensurePlatformToken(); + return new GraphQLClient(`${this.evaultServerUri}/graphql`, { + headers: { + Authorization: `Bearer ${token}`, + "X-ENAME": eName, + }, + fetch: this.fetchImpl, + }); + } + + private unwrap( + mutationName: string, + result: MetaEnvelopeMutationResult, + ): string { + const { metaEnvelope, errors } = result; + if (errors?.length) { + throw new Error(errors.map((e) => e.message).join("; ")); + } + if (!metaEnvelope) { + throw new Error(`${mutationName}: no metaEnvelope returned`); + } + return metaEnvelope.id; + } + + /** Writes one commit as a MetaEnvelope into `eName`'s eVault. Returns the new envelope's id. */ + async writeCommit( + eName: string, + payload: CommitEnvelopePayload, + acl: string[], + ): Promise { + const client = await this.getClient(eName); + const result = await client.request<{ + createMetaEnvelope: MetaEnvelopeMutationResult; + }>(CREATE_MUTATION, { + input: { + ontology: CODE_COMMIT_ONTOLOGY_ID, + payload, + acl, + }, + }); + return this.unwrap("createMetaEnvelope", result.createMetaEnvelope); + } + + /** + * Writes a repo's full snapshot into the owner's eVault: creates a new + * `repoSnapshot` MetaEnvelope the first time a given repo is seen, then + * updates that same envelope in place on every later push - "replaces + * whenever anyone makes a commit" (the owner's own words), not one + * envelope per push. `existingEnvelopeId` comes from + * `RepoEnvelopeStore` (see repoEnvelopeStore.ts), the caller's own record + * of which envelope, if any, already holds this repo's snapshot - this + * method itself has no way to look that up. + */ + async writeRepoSnapshot( + eName: string, + payload: RepoSnapshotEnvelopePayload, + acl: string[], + existingEnvelopeId: string | null, + ): Promise { + const client = await this.getClient(eName); + const input = { ontology: REPO_SNAPSHOT_ONTOLOGY_ID, payload, acl }; + + if (existingEnvelopeId) { + const result = await client.request<{ + updateMetaEnvelope: MetaEnvelopeMutationResult; + }>(UPDATE_MUTATION, { id: existingEnvelopeId, input }); + return this.unwrap("updateMetaEnvelope", result.updateMetaEnvelope); + } + + const result = await client.request<{ + createMetaEnvelope: MetaEnvelopeMutationResult; + }>(CREATE_MUTATION, { input }); + return this.unwrap("createMetaEnvelope", result.createMetaEnvelope); + } +} diff --git a/services/forgejo-code-sync/src/identity.test.ts b/services/forgejo-code-sync/src/identity.test.ts new file mode 100644 index 000000000..8209ac68a --- /dev/null +++ b/services/forgejo-code-sync/src/identity.test.ts @@ -0,0 +1,188 @@ +import { describe, expect, it, vi } from "vitest"; +import { IdentityResolver, enameFromLoginName } from "./identity.js"; + +describe("enameFromLoginName", () => { + it("returns the ename when login_name starts with @", () => { + expect(enameFromLoginName("@alice")).toBe("@alice"); + expect(enameFromLoginName("@user-a.w3id")).toBe("@user-a.w3id"); + }); + + it("returns null for an ordinary password account's login_name", () => { + expect(enameFromLoginName("alice")).toBeNull(); + }); + + it("returns null for an empty string", () => { + expect(enameFromLoginName("")).toBeNull(); + }); + + it("returns null when @ appears but not as the first character", () => { + // Not a real GitW3 login_name shape, but the check is a strict prefix + // check, not a "contains @" check - worth pinning down explicitly. + expect(enameFromLoginName("foo@bar")).toBeNull(); + }); +}); + +function jsonResponse(status: number, body: unknown): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +describe("IdentityResolver.resolveEname", () => { + it("resolves a linked account's ename", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(jsonResponse(200, { login_name: "@alice" })); + const resolver = new IdentityResolver({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + fetchImpl, + }); + + expect(await resolver.resolveEname("alice")).toBe("@alice"); + expect(fetchImpl).toHaveBeenCalledWith( + "https://git.example.org/api/v1/users/alice", + { headers: { Authorization: "token admin-token" } }, + ); + }); + + it("returns null for an account with no linked eVault", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(jsonResponse(200, { login_name: "" })); + const resolver = new IdentityResolver({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + fetchImpl, + }); + + expect(await resolver.resolveEname("bob")).toBeNull(); + }); + + it("returns null when login_name is absent from the response entirely, not just empty", async () => { + // Confirmed live against a real GitW3 instance, not assumed: Go's + // `json:"login_name,omitempty"` means a password-registered account's + // GET /users/{username} response omits the key outright rather than + // sending `"login_name": ""` - `{ login_name: "" }` above is not the + // only shape a "no linked eVault" account actually takes on the wire. + const fetchImpl = vi + .fn() + .mockResolvedValue(jsonResponse(200, { id: 3, login: "bob" })); + const resolver = new IdentityResolver({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + fetchImpl, + }); + + expect(await resolver.resolveEname("bob")).toBeNull(); + }); + + it("does not re-fetch within the TTL", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(jsonResponse(200, { login_name: "@alice" })); + const resolver = new IdentityResolver({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + ttlMs: 1000, + fetchImpl, + }); + + const now = Date.now(); + await resolver.resolveEname("alice", now); + await resolver.resolveEname("alice", now + 500); + + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it("re-fetches once the TTL has elapsed", async () => { + // mockImplementation, not mockResolvedValue: a Response body can only + // be read once, so reusing the same instance across calls would break + // the second read rather than testing anything meaningful. + const fetchImpl = vi + .fn() + .mockImplementation(async () => + jsonResponse(200, { login_name: "@alice" }), + ); + const resolver = new IdentityResolver({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + ttlMs: 1000, + fetchImpl, + }); + + const now = Date.now(); + await resolver.resolveEname("alice", now); + await resolver.resolveEname("alice", now + 1001); + + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + + it("caches a negative (no linked eVault) result too", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(jsonResponse(200, { login_name: "" })); + const resolver = new IdentityResolver({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + fetchImpl, + }); + + const now = Date.now(); + await resolver.resolveEname("bob", now); + await resolver.resolveEname("bob", now + 1); + + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it("evicts a previously-cached entry on 404 rather than retrying it later", async () => { + const fetchImpl = vi + .fn() + .mockImplementationOnce(async () => + jsonResponse(200, { login_name: "@alice" }), + ) + .mockImplementationOnce(async () => + jsonResponse(404, { message: "not found" }), + ) + .mockImplementationOnce(async () => + jsonResponse(200, { login_name: "@alice" }), + ); + const resolver = new IdentityResolver({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + // Short TTL: the second call must land past it, or it would be + // served from cache and never reach the queued 404 response at all. + ttlMs: 10, + fetchImpl, + }); + + const now = Date.now(); + expect(await resolver.resolveEname("alice", now)).toBe("@alice"); + // Past the TTL - a genuine re-fetch, landing on the account-deleted 404. + expect(await resolver.resolveEname("alice", now + 20)).toBeNull(); + // The 404 evicted the cache entry rather than caching the null - a call + // one millisecond later, well within what would otherwise be a fresh + // TTL window, still has nothing cached and must fetch again. If the + // 404 had merely returned null without evicting, this call would + // wrongly serve a cached negative instead of hitting the stub a third + // time. + expect(await resolver.resolveEname("alice", now + 21)).toBe("@alice"); + expect(fetchImpl).toHaveBeenCalledTimes(3); + }); + + it("throws on a non-404 error response, distinctly from returning null", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(jsonResponse(500, { message: "boom" })); + const resolver = new IdentityResolver({ + forgejoApiUrl: "https://git.example.org", + adminToken: "admin-token", + fetchImpl, + }); + + // Must throw, not resolve to null - a 500 is "couldn't check right now, + // retry", which the queue's backoff handles, not "no linked eVault". + await expect(resolver.resolveEname("alice")).rejects.toThrow(/500/); + }); +}); diff --git a/services/forgejo-code-sync/src/identity.ts b/services/forgejo-code-sync/src/identity.ts new file mode 100644 index 000000000..828b3ac67 --- /dev/null +++ b/services/forgejo-code-sync/src/identity.ts @@ -0,0 +1,105 @@ +/** + * Whether a GitW3 account's `login_name` is a W3DS ename, and if so, what it is. + * + * Kept as a one-line pure function, isolated from the admin-API call that + * surrounds it (see `resolveEname`), so the one rule this whole design leans + * on - login_name always begins with "@" when it was set by the w3ds-oidc-bridge, + * because claims.ts's buildClaims sets `sub: ename` verbatim and Forgejo's OAuth2 + * callback writes LoginName = gothUser.UserID (= the ID token's `sub`) unchanged - + * can be tested without any network access at all. + * + * A password-registered account's login_name is not an ename and never starts + * with "@" - that account simply has no linked eVault, which is the ordinary + * case, not a failure. See docs/superpowers/specs/2026-08-14-forgejo-code-sync-design.md + * ("Identity resolution: pusher -> eName"). + */ +export function enameFromLoginName(loginName: string): string | null { + return loginName.startsWith("@") ? loginName : null; +} + +interface CacheEntry { + ename: string | null; + expiresAt: number; +} + +export interface IdentityResolverOptions { + forgejoApiUrl: string; + /** PAT on a dedicated site-admin service account - see the spec's Trust model. */ + adminToken: string; + /** How long a resolved (or negative) result is trusted before re-fetching. */ + ttlMs?: number; + fetchImpl?: typeof fetch; +} + +const DEFAULT_TTL_MS = 60 * 60 * 1000; + +/** + * Resolves a GitW3 username to its eName via `GET /api/v1/users/{username}`, + * cached to avoid an admin-authenticated round trip on every push. + * + * `login_name` is only populated by that endpoint for an admin-or-self caller + * (`services/convert/user.go`'s `toUser`, `authed = doer.ID == user.ID || + * doer.IsAdmin`) - confirmed against GitW3's own source, not assumed. A + * `read:user`-scoped token belonging to a non-admin account gets back + * `login_name: ""` for anyone but itself, which is why `FORGEJO_ADMIN_TOKEN` + * must belong to a site-admin account, not merely carry that scope. + */ +export class IdentityResolver { + private readonly cache = new Map(); + private readonly forgejoApiUrl: string; + private readonly adminToken: string; + private readonly ttlMs: number; + private readonly fetchImpl: typeof fetch; + + constructor(options: IdentityResolverOptions) { + this.forgejoApiUrl = options.forgejoApiUrl; + this.adminToken = options.adminToken; + this.ttlMs = options.ttlMs ?? DEFAULT_TTL_MS; + this.fetchImpl = options.fetchImpl ?? fetch; + } + + /** + * Resolves a username to its eName, or `null` if the account has no linked + * W3DS identity - the ordinary case for most GitW3 accounts, not a failure. + * Throws on anything that looks like a transient infrastructure problem + * (network error, non-404 non-2xx response), so the queue's drain loop can + * tell "no eVault, skip" apart from "couldn't check right now, retry" - see + * the spec's "Identity resolution" section. + */ + async resolveEname( + username: string, + now = Date.now(), + ): Promise { + const cached = this.cache.get(username); + if (cached && cached.expiresAt > now) { + return cached.ename; + } + + const url = `${this.forgejoApiUrl}/api/v1/users/${encodeURIComponent(username)}`; + const res = await this.fetchImpl(url, { + headers: { Authorization: `token ${this.adminToken}` }, + }); + + if (res.status === 404) { + // The account no longer exists. Evicted rather than cached: unlike + // the stable "this account has no linked eVault" fact cached below, + // a 404 isn't something to keep trusting for an hour - and treating + // it as a transient failure to retry would be wrong too, since a + // deleted account isn't coming back. + this.cache.delete(username); + return null; + } + + if (!res.ok) { + throw new Error( + `GET /api/v1/users/${username} failed: HTTP ${res.status}`, + ); + } + + const body = (await res.json()) as { login_name?: string }; + const ename = enameFromLoginName(body.login_name ?? ""); + + this.cache.set(username, { ename, expiresAt: now + this.ttlMs }); + return ename; + } +} diff --git a/services/forgejo-code-sync/src/index.ts b/services/forgejo-code-sync/src/index.ts new file mode 100644 index 000000000..1517ae85e --- /dev/null +++ b/services/forgejo-code-sync/src/index.ts @@ -0,0 +1,230 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { createApp } from "./app.js"; +import { ConfigError, getConfig } from "./config.js"; +import { createArchiveFetcher } from "./content/archive.js"; +import { createDiffFetcher } from "./content/diff.js"; +import { EVaultClient } from "./evault/client.js"; +import { IdentityResolver } from "./identity.js"; +import { Queue } from "./queue.js"; +import { RepoEnvelopeStore } from "./repoEnvelopeStore.js"; +import { + type SnapshotDrainOutcome, + drainSnapshotsOnce, +} from "./snapshotSync.js"; +import { S3Storage } from "./storage/s3.js"; +import { type DrainOutcome, drainOnce } from "./sync.js"; +import type { CommitSyncTask, RepoSnapshotTask } from "./task.js"; + +/** + * How often the queues are drained. A task's own backoff (queue.ts) governs when + * an individual retry is due; this just bounds how long a freshly-queued or + * newly-due task waits before the next sweep picks it up. + */ +const DRAIN_INTERVAL_MS = 5_000; + +function logOutcome( + taskId: string, + task: CommitSyncTask, + outcome: DrainOutcome, +): void { + const label = `${task.repoFullName}@${task.commitId.slice(0, 12)}`; + switch (outcome.kind) { + case "succeeded": + console.log(`[sync] ${label} -> envelope ${outcome.envelopeId}`); + break; + case "skipped": + // The ordinary case for most GitW3 accounts, not a failure - see + // the spec's "Identity resolution" section. Must read distinctly + // from a "failed" log line, never look the same. + console.log( + `[sync] ${label} skipped - no linked eVault for pusher "${task.pusherLogin}"`, + ); + break; + case "failed": + if (outcome.status === "exhausted") { + // Needs a human. This is the one outcome that must not be + // mistaken for a routine skip or a retry still in flight. + console.error( + `[sync] EXHAUSTED task ${taskId} (${label}), needs attention: ${String(outcome.error)}`, + ); + } else { + console.warn( + `[sync] ${label} failed, retrying: ${String(outcome.error)}`, + ); + } + break; + } +} + +/** + * Same shape as logOutcome above, deliberately - the owner-snapshot sync's + * skip/retry/exhausted outcomes must read exactly as distinctly as the + * per-commit sync's do, per the same "never look the same as an error" + * requirement the spec's Delivery reliability section states for the + * commit path. + */ +function logSnapshotOutcome( + taskId: string, + task: RepoSnapshotTask, + outcome: SnapshotDrainOutcome, +): void { + const label = `${task.repoFullName}@${task.headCommitId.slice(0, 12)}`; + switch (outcome.kind) { + case "succeeded": + console.log( + `[snapshot] ${label} -> envelope ${outcome.envelopeId}`, + ); + break; + case "skipped": + // Covers both an ordinary unlinked owner and an org-owned repo - + // see snapshotSync.ts's processSnapshotTask for why both fall out + // of the same identity resolution with no separate check. + console.log( + `[snapshot] ${label} skipped - no linked eVault for owner "${task.ownerLogin}"`, + ); + break; + case "failed": + if (outcome.status === "exhausted") { + console.error( + `[snapshot] EXHAUSTED task ${taskId} (${label}), needs attention: ${String(outcome.error)}`, + ); + } else { + console.warn( + `[snapshot] ${label} failed, retrying: ${String(outcome.error)}`, + ); + } + break; + } +} + +async function main(): Promise { + // Anything wrong with the environment stops the process here, rather than + // surfacing as a silently-unresolved eName on the first push, when the + // symptom no longer points at the cause - matching the bridge's own + // config.ts. + const config = getConfig(); + + const here = path.dirname(fileURLToPath(import.meta.url)); + // Where the retry queue persists in a real deployment is an open item tied + // to that deployment's own storage - see the plan's Phase 5. This default + // is for local development, next to the package rather than inside src/ + // or dist/ so it survives a rebuild. + const queueDir = path.resolve(here, "../.queue"); + // A second, independent queue - see task.ts's RepoSnapshotTask and + // webhook/push.ts for why this can't share the commit queue's contents + // (different task shape, different granularity - one per push, not one + // per commit). + const snapshotQueueDir = path.resolve(here, "../.queue-snapshots"); + // repoFullName -> envelopeId, so a later push updates the same + // repoSnapshot envelope in place instead of creating a new one - see + // repoEnvelopeStore.ts. + const repoEnvelopeStoreDir = path.resolve(here, "../.repo-envelopes"); + + const queue = new Queue({ dir: queueDir }); + await queue.init(); + + const snapshotQueue = new Queue({ + dir: snapshotQueueDir, + }); + await snapshotQueue.init(); + + const repoEnvelopeStore = new RepoEnvelopeStore({ + dir: repoEnvelopeStoreDir, + }); + await repoEnvelopeStore.init(); + + const identity = new IdentityResolver({ + forgejoApiUrl: config.forgejoApiUrl, + adminToken: config.forgejoAdminToken, + }); + + const evault = new EVaultClient({ + registryUrl: config.registryUrl, + evaultServerUri: config.evaultServerUri, + publicUrl: config.publicUrl, + }); + + const storage = new S3Storage(config.s3); + + const fetchDiff = createDiffFetcher({ + forgejoApiUrl: config.forgejoApiUrl, + adminToken: config.forgejoAdminToken, + storage, + }); + + const fetchArchive = createArchiveFetcher({ + forgejoApiUrl: config.forgejoApiUrl, + adminToken: config.forgejoAdminToken, + storage, + }); + + const app = createApp({ + queue, + snapshotQueue, + webhookSecret: config.webhookSecret, + }); + + const server = app.listen(config.port, () => { + console.log(`forgejo-code-sync listening on :${config.port}`); + console.log(` forgejo ${config.forgejoApiUrl}`); + console.log(` registry ${config.registryUrl}`); + console.log(` evault ${config.evaultServerUri}`); + console.log(` s3 ${config.s3.bucket} (${config.s3.endpoint})`); + console.log(` queue ${queueDir}`); + console.log(` queue ${snapshotQueueDir} (repo snapshots)`); + }); + + // Guards against overlapping drains: if a sweep is still running (e.g. a + // slow eVault) when the next tick fires, that tick is skipped rather than + // starting a second pass over the same due tasks - two concurrent drains + // could otherwise both pick up the same task before either marks it done. + // Both queues share one guard and one interval: they're independent + // stores, but there's no reason to run two separate timers for what's + // conceptually one "drain everything that's due" tick. + let draining = false; + const timer = setInterval(() => { + if (draining) return; + draining = true; + Promise.all([ + drainOnce({ + queue, + identity, + evault, + fetchDiff, + onOutcome: logOutcome, + }), + drainSnapshotsOnce({ + queue: snapshotQueue, + identity, + evault, + fetchArchive, + store: repoEnvelopeStore, + onOutcome: logSnapshotOutcome, + }), + ]) + .catch((error: unknown) => { + console.error("[sync] drain failed:", error); + }) + .finally(() => { + draining = false; + }); + }, DRAIN_INTERVAL_MS); + timer.unref(); + + const shutdown = () => { + clearInterval(timer); + server.close(() => process.exit(0)); + }; + process.on("SIGTERM", shutdown); + process.on("SIGINT", shutdown); +} + +main().catch((error: unknown) => { + if (error instanceof ConfigError) { + console.error(`forgejo-code-sync cannot start: ${error.message}`); + process.exit(1); + } + console.error(error); + process.exit(1); +}); diff --git a/services/forgejo-code-sync/src/queue.test.ts b/services/forgejo-code-sync/src/queue.test.ts new file mode 100644 index 000000000..5a4d89123 --- /dev/null +++ b/services/forgejo-code-sync/src/queue.test.ts @@ -0,0 +1,157 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { Queue } from "./queue.js"; + +interface TestPayload { + commitId: string; +} + +let dir: string; + +beforeEach(async () => { + dir = await mkdtemp(path.join(tmpdir(), "forgejo-code-sync-queue-")); +}); + +afterEach(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +describe("Queue", () => { + it("enqueues a task as pending, immediately due", async () => { + const queue = new Queue({ dir }); + await queue.init(); + + const id = await queue.enqueue({ commitId: "abc123" }); + + const due = await queue.due(); + expect(due).toHaveLength(1); + expect(due[0]?.id).toBe(id); + expect(due[0]?.status).toBe("pending"); + expect(due[0]?.attempts).toBe(0); + }); + + it("removes a task on success", async () => { + const queue = new Queue({ dir }); + await queue.init(); + const id = await queue.enqueue({ commitId: "abc123" }); + + await queue.markSucceeded(id); + + expect(await queue.list()).toEqual([]); + }); + + it("removes a task on skip, distinctly from a failure", async () => { + const queue = new Queue({ dir }); + await queue.init(); + const id = await queue.enqueue({ commitId: "abc123" }); + + await queue.markSkipped(id); + + // A skipped task is gone, not lingering in any retryable status - it + // must never be confused with something still pending or retrying. + expect(await queue.list()).toEqual([]); + }); + + it("retries a failed task with backoff, not dropping it", async () => { + const queue = new Queue({ + dir, + maxAttempts: 5, + baseDelayMs: 1000, + }); + await queue.init(); + const id = await queue.enqueue({ commitId: "abc123" }); + + const now = Date.now(); + const status = await queue.markFailed( + id, + new Error("eVault down"), + now, + ); + + expect(status).toBe("retrying"); + const [task] = await queue.list(); + expect(task?.status).toBe("retrying"); + expect(task?.attempts).toBe(1); + expect(task?.lastError).toBe("eVault down"); + // Backoff: baseDelayMs * 2^(attempts-1) = 1000 * 2^0 = 1000 + expect(task?.nextAttemptAt).toBe(now + 1000); + + // Not due yet - the queue must not hand back a task before its backoff + // has elapsed. + expect(await queue.due(now)).toEqual([]); + expect(await queue.due(now + 1000)).toHaveLength(1); + }); + + it("increases the backoff exponentially across repeated failures", async () => { + const queue = new Queue({ + dir, + maxAttempts: 10, + baseDelayMs: 1000, + }); + await queue.init(); + const id = await queue.enqueue({ commitId: "abc123" }); + + const now = Date.now(); + await queue.markFailed(id, "err", now); + await queue.markFailed(id, "err", now); + const status = await queue.markFailed(id, "err", now); + + expect(status).toBe("retrying"); + const [task] = await queue.list(); + expect(task?.attempts).toBe(3); + // baseDelayMs * 2^(3-1) = 1000 * 4 = 4000 + expect(task?.nextAttemptAt).toBe(now + 4000); + }); + + it("marks a task exhausted, not silently removed, once retries run out", async () => { + const queue = new Queue({ dir, maxAttempts: 2 }); + await queue.init(); + const id = await queue.enqueue({ commitId: "abc123" }); + + await queue.markFailed(id, "err"); + const status = await queue.markFailed(id, "final failure"); + + expect(status).toBe("exhausted"); + const [task] = await queue.list(); + expect(task).toBeDefined(); + expect(task?.status).toBe("exhausted"); + expect(task?.lastError).toBe("final failure"); + }); + + it("never hands an exhausted task back from due()", async () => { + const queue = new Queue({ dir, maxAttempts: 1 }); + await queue.init(); + const id = await queue.enqueue({ commitId: "abc123" }); + + await queue.markFailed(id, "err"); + + expect(await queue.due()).toEqual([]); + // But it is still findable via list() - exhausted tasks need a human, + // not to vanish. + expect(await queue.list()).toHaveLength(1); + }); + + it("survives a simulated restart - a new Queue instance sees prior state", async () => { + const first = new Queue({ dir }); + await first.init(); + const id = await first.enqueue({ commitId: "abc123" }); + + // Simulate a process restart: a brand new Queue instance, same dir, no + // in-memory state carried over. + const second = new Queue({ dir }); + const due = await second.due(); + + expect(due).toHaveLength(1); + expect(due[0]?.id).toBe(id); + }); + + it("returns an empty list when the directory has not been initialised yet", async () => { + const queue = new Queue({ + dir: path.join(dir, "not-created"), + }); + expect(await queue.list()).toEqual([]); + expect(await queue.due()).toEqual([]); + }); +}); diff --git a/services/forgejo-code-sync/src/queue.ts b/services/forgejo-code-sync/src/queue.ts new file mode 100644 index 000000000..9755f0436 --- /dev/null +++ b/services/forgejo-code-sync/src/queue.ts @@ -0,0 +1,168 @@ +import { randomUUID } from "node:crypto"; +import { + mkdir, + readFile, + readdir, + rename, + rm, + writeFile, +} from "node:fs/promises"; +import path from "node:path"; + +export type QueueTaskStatus = "pending" | "retrying" | "exhausted"; + +export interface QueueTask { + id: string; + payload: T; + status: QueueTaskStatus; + attempts: number; + /** Epoch ms. The task is not picked up by `due()` before this time. */ + nextAttemptAt: number; + lastError?: string; + createdAt: number; +} + +export interface QueueOptions { + /** Where task files are persisted. Must survive a process restart. */ + dir: string; + maxAttempts?: number; + /** Base delay for exponential backoff; doubled per attempt. */ + baseDelayMs?: number; +} + +const DEFAULT_MAX_ATTEMPTS = 5; +const DEFAULT_BASE_DELAY_MS = 30_000; + +/** + * A persisted retry queue for commit-sync tasks. + * + * Exists because Forgejo has no automatic retry/redelivery of failed webhook + * deliveries at all (confirmed against GitW3's `services/webhook/deliver.go` - + * it records success/failure and stops; the only resend is a human clicking + * "Replay" in the webhook history UI). So this service's own retry is the only + * safety net a dropped delivery gets - see the spec's "Delivery reliability" + * section. Every operation reads or writes the backing directory directly, with + * no separate in-memory cache, so the queue's state IS the disk: a task queued + * before a crash or restart is still there afterward, with no reload step needed. + * + * A task's terminal states are asymmetric on purpose. `markSucceeded` and + * `markSkipped` both remove the task file - the work is done, and there is + * nothing further to act on. `markFailed` keeps retrying with backoff while + * attempts remain, but once exhausted the task is left on disk in the + * "exhausted" status rather than removed: it needs a human, and staying + * findable is what makes that possible. Silently deleting it would reproduce + * exactly the invisible-data-loss failure mode this queue exists to prevent. + */ +export class Queue { + private readonly dir: string; + private readonly maxAttempts: number; + private readonly baseDelayMs: number; + + constructor(options: QueueOptions) { + this.dir = options.dir; + this.maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS; + this.baseDelayMs = options.baseDelayMs ?? DEFAULT_BASE_DELAY_MS; + } + + async init(): Promise { + await mkdir(this.dir, { recursive: true }); + } + + private filePath(id: string): string { + return path.join(this.dir, `${id}.json`); + } + + private async writeTask(task: QueueTask): Promise { + // Write-then-rename so a crash mid-write can never leave a half-written, + // unparseable task file behind - the rename is atomic on the same + // filesystem, the write alone is not. + const tmpPath = `${this.filePath(task.id)}.tmp`; + await writeFile(tmpPath, JSON.stringify(task, null, 2)); + await rename(tmpPath, this.filePath(task.id)); + } + + private async readTask(id: string): Promise> { + const raw = await readFile(this.filePath(id), "utf8"); + return JSON.parse(raw) as QueueTask; + } + + async enqueue(payload: T, now = Date.now()): Promise { + const task: QueueTask = { + id: randomUUID(), + payload, + status: "pending", + attempts: 0, + nextAttemptAt: now, + createdAt: now, + }; + await this.writeTask(task); + return task.id; + } + + /** Every task currently persisted, in any status. For introspection and tests. */ + async list(): Promise[]> { + let files: string[]; + try { + files = await readdir(this.dir); + } catch { + return []; + } + const tasks: QueueTask[] = []; + for (const file of files) { + if (!file.endsWith(".json")) continue; + const raw = await readFile(path.join(this.dir, file), "utf8"); + tasks.push(JSON.parse(raw) as QueueTask); + } + return tasks; + } + + /** Pending or retrying tasks whose backoff has elapsed - ready to process now. */ + async due(now = Date.now()): Promise[]> { + const tasks = await this.list(); + return tasks.filter( + (task) => + (task.status === "pending" || task.status === "retrying") && + task.nextAttemptAt <= now, + ); + } + + async markSucceeded(id: string): Promise { + await rm(this.filePath(id), { force: true }); + } + + /** + * The task was resolved without ever being attempted against the eVault - + * e.g. the pusher has no linked identity. Removed the same as a success: it + * is done, not failed, and must not be confused with something still + * pending or retrying by anything inspecting the queue's contents. + */ + async markSkipped(id: string): Promise { + await rm(this.filePath(id), { force: true }); + } + + /** + * The task failed. Rescheduled with exponential backoff while attempts + * remain; left in "exhausted" status, on disk, once they don't. Returns the + * resulting status so callers can decide how loudly to log it. + */ + async markFailed( + id: string, + error: unknown, + now = Date.now(), + ): Promise { + const task = await this.readTask(id); + task.attempts += 1; + task.lastError = error instanceof Error ? error.message : String(error); + + if (task.attempts >= this.maxAttempts) { + task.status = "exhausted"; + } else { + task.status = "retrying"; + task.nextAttemptAt = + now + this.baseDelayMs * 2 ** (task.attempts - 1); + } + + await this.writeTask(task); + return task.status; + } +} diff --git a/services/forgejo-code-sync/src/repoEnvelopeStore.test.ts b/services/forgejo-code-sync/src/repoEnvelopeStore.test.ts new file mode 100644 index 000000000..46a6dec55 --- /dev/null +++ b/services/forgejo-code-sync/src/repoEnvelopeStore.test.ts @@ -0,0 +1,52 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { RepoEnvelopeStore } from "./repoEnvelopeStore.js"; + +let dir: string; +let store: RepoEnvelopeStore; + +beforeEach(async () => { + dir = await mkdtemp(path.join(tmpdir(), "forgejo-code-sync-repo-store-")); + store = new RepoEnvelopeStore({ dir }); + await store.init(); +}); + +afterEach(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +describe("RepoEnvelopeStore", () => { + it("returns null for a repo with no recorded envelope", async () => { + expect(await store.get("alice/repo")).toBeNull(); + }); + + it("returns the recorded envelope id after set", async () => { + await store.set("alice/repo", "envelope-1"); + expect(await store.get("alice/repo")).toBe("envelope-1"); + }); + + it("overwrites the recorded envelope id on a second set for the same repo", async () => { + await store.set("alice/repo", "envelope-1"); + await store.set("alice/repo", "envelope-2"); + expect(await store.get("alice/repo")).toBe("envelope-2"); + }); + + it("keeps different repos' envelope ids independent", async () => { + await store.set("alice/repo", "envelope-1"); + await store.set("bob/other-repo", "envelope-2"); + + expect(await store.get("alice/repo")).toBe("envelope-1"); + expect(await store.get("bob/other-repo")).toBe("envelope-2"); + }); + + it("survives a simulated restart - reload from the backing store, not memory", async () => { + await store.set("alice/repo", "envelope-1"); + + const reloaded = new RepoEnvelopeStore({ dir }); + await reloaded.init(); + + expect(await reloaded.get("alice/repo")).toBe("envelope-1"); + }); +}); diff --git a/services/forgejo-code-sync/src/repoEnvelopeStore.ts b/services/forgejo-code-sync/src/repoEnvelopeStore.ts new file mode 100644 index 000000000..bd676a16e --- /dev/null +++ b/services/forgejo-code-sync/src/repoEnvelopeStore.ts @@ -0,0 +1,67 @@ +import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import path from "node:path"; + +/** + * Durable `repoFullName -> envelopeId` mapping, so a later push can find the + * one existing `repoSnapshot` MetaEnvelope for a repo and update it in place + * (see evault/client.ts's `writeRepoSnapshot`) instead of creating a new one + * every time - "replaces whenever anyone makes a commit" (the owner's own + * words) means one envelope per repo, not one per push. + * + * Deliberately NOT `evault-core`'s `metaEnvelopes` list query: that query has + * a confirmed, live, currently-unfixed ACL-filtering bug (see the spec's + * Testing section) - unrelated to this store's own correctness, but using it + * to "search for the existing envelope" would add an extra live dependency + * and inherit a bug this service doesn't need to depend on when a local + * mapping is simpler and already matches queue.ts's own persistence style: + * one file per key, write-then-rename so a crash mid-write never leaves a + * half-written, unparseable file behind. + */ +export class RepoEnvelopeStore { + private readonly dir: string; + + constructor(options: { dir: string }) { + this.dir = options.dir; + } + + async init(): Promise { + await mkdir(this.dir, { recursive: true }); + } + + /** + * repoFullName is "owner/name" - not filesystem-safe on its own (the "/" + * would be read as a path separator), so it's flattened into one segment + * the same way S3Storage.buildArchiveKey sanitises its own path + * components, just joined instead of nested (a flat directory is enough + * here - there's no need for the two-level layout the S3 key uses). + */ + private filePath(repoFullName: string): string { + const safe = repoFullName.replace(/[^\w.-]/g, "_"); + return path.join(this.dir, `${safe}.json`); + } + + /** The existing envelope id for this repo, or `null` if none has been recorded yet. */ + async get(repoFullName: string): Promise { + try { + const raw = await readFile(this.filePath(repoFullName), "utf8"); + const parsed = JSON.parse(raw) as { envelopeId: string }; + return parsed.envelopeId; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return null; + } + throw error; + } + } + + /** Records (or overwrites) the envelope id a repo's snapshot lives at. */ + async set(repoFullName: string, envelopeId: string): Promise { + const filePath = this.filePath(repoFullName); + const tmpPath = `${filePath}.tmp`; + await writeFile( + tmpPath, + JSON.stringify({ repoFullName, envelopeId }, null, 2), + ); + await rename(tmpPath, filePath); + } +} diff --git a/services/forgejo-code-sync/src/snapshotSync.test.ts b/services/forgejo-code-sync/src/snapshotSync.test.ts new file mode 100644 index 000000000..037750d7c --- /dev/null +++ b/services/forgejo-code-sync/src/snapshotSync.test.ts @@ -0,0 +1,258 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { + EVaultClient, + RepoSnapshotEnvelopePayload, +} from "./evault/client.js"; +import type { IdentityResolver } from "./identity.js"; +import { Queue } from "./queue.js"; +import { RepoEnvelopeStore } from "./repoEnvelopeStore.js"; +import { type SnapshotDrainDeps, processSnapshotTask } from "./snapshotSync.js"; +import type { RepoSnapshotTask } from "./task.js"; + +let dir: string; +let storeDir: string; +let queue: Queue; +let store: RepoEnvelopeStore; + +const baseTask: RepoSnapshotTask = { + repoFullName: "alice/repo", + repoPrivate: false, + ref: "refs/heads/main", + ownerLogin: "alice", + headCommitId: "abc123", +}; + +beforeEach(async () => { + dir = await mkdtemp(path.join(tmpdir(), "forgejo-code-sync-snapshot-")); + storeDir = await mkdtemp( + path.join(tmpdir(), "forgejo-code-sync-snapshot-store-"), + ); + queue = new Queue({ dir }); + await queue.init(); + store = new RepoEnvelopeStore({ dir: storeDir }); + await store.init(); +}); + +afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + await rm(storeDir, { recursive: true, force: true }); +}); + +function makeDeps( + overrides: Partial = {}, +): SnapshotDrainDeps { + const identity = { + resolveEname: vi.fn().mockResolvedValue("@alice"), + } as unknown as IdentityResolver; + + const evault = { + writeRepoSnapshot: vi.fn().mockResolvedValue("snapshot-envelope-1"), + } as unknown as EVaultClient; + + const fetchArchive = vi + .fn() + .mockResolvedValue("https://s3.example.org/repos/alice/repo.zip"); + + return { + queue, + identity, + evault, + fetchArchive, + store, + now: () => new Date("2026-08-15T12:00:00.000Z"), + ...overrides, + }; +} + +describe("processSnapshotTask", () => { + it('writes with acl ["*"] for a public repo, creating (no existing envelope id)', async () => { + const id = await queue.enqueue(baseTask); + const deps = makeDeps(); + + const outcome = await processSnapshotTask(id, baseTask, deps); + + expect(outcome).toEqual({ + kind: "succeeded", + envelopeId: "snapshot-envelope-1", + }); + expect(deps.evault.writeRepoSnapshot).toHaveBeenCalledWith( + "@alice", + expect.objectContaining({ + repo: "alice/repo", + headCommitId: "abc123", + ownerEName: "@alice", + updatedAt: "2026-08-15T12:00:00.000Z", + }), + ["*"], + null, + ); + expect(await queue.list()).toEqual([]); + }); + + it("writes with an owner-only acl for a private repo", async () => { + const privateTask = { ...baseTask, repoPrivate: true }; + const id = await queue.enqueue(privateTask); + const deps = makeDeps(); + + await processSnapshotTask(id, privateTask, deps); + + expect(deps.evault.writeRepoSnapshot).toHaveBeenCalledWith( + "@alice", + expect.anything(), + ["@alice"], + null, + ); + }); + + it("passes the existing envelope id through when this repo already has a recorded snapshot - update, not create", async () => { + await store.set("alice/repo", "existing-envelope-id"); + const id = await queue.enqueue(baseTask); + const deps = makeDeps(); + + await processSnapshotTask(id, baseTask, deps); + + expect(deps.evault.writeRepoSnapshot).toHaveBeenCalledWith( + "@alice", + expect.anything(), + ["*"], + "existing-envelope-id", + ); + }); + + it("records the returned envelope id in the store after a successful write", async () => { + const id = await queue.enqueue(baseTask); + const deps = makeDeps(); + + await processSnapshotTask(id, baseTask, deps); + + expect(await store.get("alice/repo")).toBe("snapshot-envelope-1"); + }); + + it("marks an unlinked owner's task done-and-skipped, never calling writeRepoSnapshot - covers both an ordinary unlinked account and an org-owned repo", async () => { + const id = await queue.enqueue(baseTask); + const deps = makeDeps({ + identity: { + resolveEname: vi.fn().mockResolvedValue(null), + } as unknown as IdentityResolver, + }); + + const outcome = await processSnapshotTask(id, baseTask, deps); + + expect(outcome).toEqual({ kind: "skipped" }); + expect(deps.evault.writeRepoSnapshot).not.toHaveBeenCalled(); + expect(await queue.list()).toEqual([]); + expect(await store.get("alice/repo")).toBeNull(); + }); + + it("leaves a task in the queue for retry on an eVault write failure, rather than dropping it", async () => { + const id = await queue.enqueue(baseTask); + const deps = makeDeps({ + evault: { + writeRepoSnapshot: vi + .fn() + .mockRejectedValue(new Error("eVault down")), + } as unknown as EVaultClient, + }); + + const outcome = await processSnapshotTask(id, baseTask, deps); + + expect(outcome.kind).toBe("failed"); + const [task] = await queue.list(); + expect(task?.status).toBe("retrying"); + expect(task?.lastError).toBe("eVault down"); + // Never recorded - the write never actually succeeded. + expect(await store.get("alice/repo")).toBeNull(); + }); + + it("marks the task failed, not skipped, when identity resolution itself errors", async () => { + const id = await queue.enqueue(baseTask); + const deps = makeDeps({ + identity: { + resolveEname: vi + .fn() + .mockRejectedValue( + new Error("GET /users failed: HTTP 500"), + ), + } as unknown as IdentityResolver, + }); + + const outcome = await processSnapshotTask(id, baseTask, deps); + + expect(outcome.kind).toBe("failed"); + expect(deps.evault.writeRepoSnapshot).not.toHaveBeenCalled(); + const [task] = await queue.list(); + expect(task?.status).toBe("retrying"); + }); + + it("leaves a task in the queue for retry when the archive fetch/upload fails", async () => { + const id = await queue.enqueue(baseTask); + const deps = makeDeps({ + fetchArchive: vi + .fn() + .mockRejectedValue(new Error("S3 unreachable")), + }); + + const outcome = await processSnapshotTask(id, baseTask, deps); + + expect(outcome.kind).toBe("failed"); + expect(deps.evault.writeRepoSnapshot).not.toHaveBeenCalled(); + const [task] = await queue.list(); + expect(task?.status).toBe("retrying"); + expect(task?.lastError).toBe("S3 unreachable"); + }); + + it("calls fetchArchive with the task itself", async () => { + const id = await queue.enqueue(baseTask); + const deps = makeDeps(); + + await processSnapshotTask(id, baseTask, deps); + + expect(deps.fetchArchive).toHaveBeenCalledWith(baseTask); + }); + + it("resolves eName from ownerLogin, not any pusher field", async () => { + const id = await queue.enqueue(baseTask); + const deps = makeDeps(); + + await processSnapshotTask(id, baseTask, deps); + + expect(deps.identity.resolveEname).toHaveBeenCalledWith("alice"); + }); + + it("calls onOutcome for every outcome kind", async () => { + const id = await queue.enqueue(baseTask); + const onOutcome = vi.fn(); + const deps = makeDeps({ onOutcome }); + + await processSnapshotTask(id, baseTask, deps); + + expect(onOutcome).toHaveBeenCalledWith( + id, + baseTask, + expect.objectContaining({ kind: "succeeded" }), + ); + }); + + it("passes the fetched snapshotUrl through to the written payload", async () => { + const id = await queue.enqueue(baseTask); + const deps = makeDeps({ + fetchArchive: vi + .fn() + .mockResolvedValue("https://example.org/archive.zip"), + }); + + await processSnapshotTask(id, baseTask, deps); + + const call = (deps.evault.writeRepoSnapshot as ReturnType) + .mock.calls[0] as [ + string, + RepoSnapshotEnvelopePayload, + string[], + string | null, + ]; + expect(call[1].snapshotUrl).toBe("https://example.org/archive.zip"); + }); +}); diff --git a/services/forgejo-code-sync/src/snapshotSync.ts b/services/forgejo-code-sync/src/snapshotSync.ts new file mode 100644 index 000000000..71b9ce6ff --- /dev/null +++ b/services/forgejo-code-sync/src/snapshotSync.ts @@ -0,0 +1,120 @@ +import { deriveAcl } from "./evault/acl.js"; +import type { + EVaultClient, + RepoSnapshotEnvelopePayload, +} from "./evault/client.js"; +import type { IdentityResolver } from "./identity.js"; +import type { Queue, QueueTaskStatus } from "./queue.js"; +import type { RepoEnvelopeStore } from "./repoEnvelopeStore.js"; +import type { RepoSnapshotTask } from "./task.js"; + +/** + * Fetches a repo archive at the push's headCommitId and uploads it to S3, + * returning the resulting URL. Throws on any failure - see content/archive.ts. + */ +export type ArchiveFetcher = (task: RepoSnapshotTask) => Promise; + +export type SnapshotDrainOutcome = + | { kind: "succeeded"; envelopeId: string } + | { kind: "skipped" } + | { kind: "failed"; status: QueueTaskStatus; error: unknown }; + +export interface SnapshotDrainDeps { + queue: Queue; + identity: IdentityResolver; + evault: EVaultClient; + fetchArchive: ArchiveFetcher; + store: RepoEnvelopeStore; + /** Injectable for deterministic tests; defaults to the real clock. */ + now?: () => Date; + onOutcome?: ( + taskId: string, + task: RepoSnapshotTask, + outcome: SnapshotDrainOutcome, + ) => void; +} + +/** + * Processes one queued repo-snapshot task through to completion: resolve the + * OWNER's eName, derive the ACL, fetch a fresh archive, then create-or-update + * that repo's one `repoSnapshot` MetaEnvelope - never a second one. + * + * Mirrors sync.ts's processTask in shape and failure semantics deliberately - + * same "skip vs retry vs exhausted" distinctions, same reasoning for why an + * unresolved eName is a skip, not a failure - but resolves the REPO OWNER + * (`task.ownerLogin`), not the pusher, and reuses the exact same + * `IdentityResolver` (it is already generic over any Forgejo username, not + * pusher-specific) and the exact same `deriveAcl` (the same repo-visibility + * signal governs both the commit envelope's ACL and this one). + * + * An org-owned repo resolves the same way an unlinked personal account does: + * `GET /api/v1/users/{orgLogin}` succeeds (Forgejo stores organizations as + * user-table rows too, and that endpoint has no user-type filter - checked + * against `routers/api/v1/user/user.go`'s `GetInfo`), but an organization + * never signs in through the bridge, so its `login_name` never starts with + * "@" and `IdentityResolver.resolveEname` naturally returns `null` for it - + * no separate "is this an organization" check needed, it falls out of the + * existing identity resolution for free. + */ +export async function processSnapshotTask( + taskId: string, + task: RepoSnapshotTask, + deps: SnapshotDrainDeps, +): Promise { + const now = deps.now ?? (() => new Date()); + try { + const eName = await deps.identity.resolveEname(task.ownerLogin); + if (!eName) { + await deps.queue.markSkipped(taskId); + const outcome: SnapshotDrainOutcome = { kind: "skipped" }; + deps.onOutcome?.(taskId, task, outcome); + return outcome; + } + + const acl = deriveAcl(task.repoPrivate, eName); + const snapshotUrl = await deps.fetchArchive(task); + const existingEnvelopeId = await deps.store.get(task.repoFullName); + + const payload: RepoSnapshotEnvelopePayload = { + repo: task.repoFullName, + ref: task.ref, + headCommitId: task.headCommitId, + ownerEName: eName, + snapshotUrl, + updatedAt: now().toISOString(), + }; + + const envelopeId = await deps.evault.writeRepoSnapshot( + eName, + payload, + acl, + existingEnvelopeId, + ); + await deps.store.set(task.repoFullName, envelopeId); + + await deps.queue.markSucceeded(taskId); + const outcome: SnapshotDrainOutcome = { + kind: "succeeded", + envelopeId, + }; + deps.onOutcome?.(taskId, task, outcome); + return outcome; + } catch (error) { + const status = await deps.queue.markFailed(taskId, error); + const outcome: SnapshotDrainOutcome = { kind: "failed", status, error }; + deps.onOutcome?.(taskId, task, outcome); + return outcome; + } +} + +/** Drains every currently-due snapshot task once. Call on an interval from index.ts. */ +export async function drainSnapshotsOnce( + deps: SnapshotDrainDeps, +): Promise { + const due = await deps.queue.due(); + const outcomes: SnapshotDrainOutcome[] = []; + for (const task of due) { + outcomes.push(await processSnapshotTask(task.id, task.payload, deps)); + } + return outcomes; +} diff --git a/services/forgejo-code-sync/src/storage/s3.test.ts b/services/forgejo-code-sync/src/storage/s3.test.ts new file mode 100644 index 000000000..04c597113 --- /dev/null +++ b/services/forgejo-code-sync/src/storage/s3.test.ts @@ -0,0 +1,152 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const sendMock = vi.fn().mockResolvedValue({}); +let capturedCommands: unknown[] = []; + +vi.mock("@aws-sdk/client-s3", () => { + class PutObjectCommand { + input: unknown; + constructor(input: unknown) { + this.input = input; + capturedCommands.push(input); + } + } + class S3Client { + send = sendMock; + } + return { S3Client, PutObjectCommand }; +}); + +const { S3Storage } = await import("./s3.js"); + +beforeEach(() => { + sendMock.mockClear(); + capturedCommands = []; +}); + +const baseOptions = { + endpoint: "https://nyc3.digitaloceanspaces.com", + region: "nyc3", + accessKeyId: "key", + secretAccessKey: "secret", + bucket: "my-bucket", +}; + +describe("S3Storage.buildKey", () => { + it("scopes the key under the eName, sanitised", () => { + expect(S3Storage.buildKey("@alice", "alice/repo", "abc123")).toBe( + "diffs/alice/alice_repo/abc123.diff", + ); + }); +}); + +describe("S3Storage.uploadDiff", () => { + it("uploads with public-read ACL for a public repo", async () => { + const storage = new S3Storage(baseOptions); + const url = await storage.uploadDiff( + "@alice", + "alice/repo", + "abc123", + "diff --git a/a.ts b/a.ts", + true, + ); + + expect(sendMock).toHaveBeenCalledTimes(1); + const input = capturedCommands[0] as Record; + expect(input.ACL).toBe("public-read"); + expect(input.Bucket).toBe("my-bucket"); + expect(url).toBe( + "https://my-bucket.nyc3.digitaloceanspaces.com/diffs/alice/alice_repo/abc123.diff", + ); + }); + + it("uploads with no public ACL for a private repo - the object must not be publicly readable", async () => { + const storage = new S3Storage(baseOptions); + await storage.uploadDiff( + "@alice", + "alice/repo", + "abc123", + "diff --git a/a.ts b/a.ts", + false, + ); + + const input = capturedCommands[0] as Record; + expect(input.ACL).toBeUndefined(); + }); + + it("uses the configured CDN URL when provided, instead of the bucket sub-domain", async () => { + const storage = new S3Storage({ + ...baseOptions, + cdnUrl: "https://cdn.example.org", + }); + const url = await storage.uploadDiff( + "@alice", + "alice/repo", + "abc123", + "diff", + true, + ); + + expect(url).toBe( + "https://cdn.example.org/diffs/alice/alice_repo/abc123.diff", + ); + }); +}); + +describe("S3Storage.buildArchiveKey", () => { + it("scopes the key under the repo, one key per repo - not per commit", () => { + expect(S3Storage.buildArchiveKey("alice/repo")).toBe( + "repos/alice/repo.zip", + ); + }); +}); + +describe("S3Storage.uploadRepoArchive", () => { + it("uploads with public-read ACL for a public repo", async () => { + const storage = new S3Storage(baseOptions); + const url = await storage.uploadRepoArchive( + "alice/repo", + Buffer.from("fake zip bytes"), + true, + ); + + expect(sendMock).toHaveBeenCalledTimes(1); + const input = capturedCommands[0] as Record; + expect(input.ACL).toBe("public-read"); + expect(input.ContentType).toBe("application/zip"); + expect(input.Bucket).toBe("my-bucket"); + expect(url).toBe( + "https://my-bucket.nyc3.digitaloceanspaces.com/repos/alice/repo.zip", + ); + }); + + it("uploads with no public ACL for a private repo - the object must not be publicly readable", async () => { + const storage = new S3Storage(baseOptions); + await storage.uploadRepoArchive( + "alice/repo", + Buffer.from("fake zip bytes"), + false, + ); + + const input = capturedCommands[0] as Record; + expect(input.ACL).toBeUndefined(); + }); + + it("reuses the same key across calls for the same repo - overwrite in place, not accumulate", async () => { + const storage = new S3Storage(baseOptions); + await storage.uploadRepoArchive( + "alice/repo", + Buffer.from("first push"), + true, + ); + await storage.uploadRepoArchive( + "alice/repo", + Buffer.from("second push"), + true, + ); + + expect(capturedCommands).toHaveLength(2); + const [first, second] = capturedCommands as Record[]; + expect(first?.Key).toBe(second?.Key); + }); +}); diff --git a/services/forgejo-code-sync/src/storage/s3.ts b/services/forgejo-code-sync/src/storage/s3.ts new file mode 100644 index 000000000..cefa707b2 --- /dev/null +++ b/services/forgejo-code-sync/src/storage/s3.ts @@ -0,0 +1,136 @@ +import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; + +/** + * Uploads commit diffs to the same DigitalOcean Spaces (S3-compatible) bucket + * evault-core's own StorageService.ts uses, so diff content is never inlined + * into an eVault MetaEnvelope - the eVault gets a link, not the blob. See + * docs/superpowers/specs/2026-08-14-forgejo-code-sync-design.md ("What gets + * written"). Uploads directly to S3, bypassing evault-core's own `uploadFile` + * GraphQL mutation deliberately: that mutation caps at 250MB + * (evault-core's `MAX_FILE_BYTES`) on top of a 350MB request-body limit, + * neither of which is "any amount" - S3 itself has no such ceiling. + */ +export interface S3StorageOptions { + endpoint: string; + region: string; + accessKeyId: string; + secretAccessKey: string; + bucket: string; + /** Public/CDN base URL; defaults to the bucket sub-domain on the endpoint. */ + cdnUrl?: string; +} + +export class S3Storage { + private readonly client: S3Client; + private readonly bucket: string; + private readonly cdnBaseUrl: string; + + constructor(options: S3StorageOptions) { + this.bucket = options.bucket; + this.cdnBaseUrl = ( + options.cdnUrl || + options.endpoint.replace("https://", `https://${options.bucket}.`) + ).replace(/\/$/, ""); + + this.client = new S3Client({ + endpoint: options.endpoint, + region: options.region, + forcePathStyle: false, + credentials: { + accessKeyId: options.accessKeyId, + secretAccessKey: options.secretAccessKey, + }, + }); + } + + /** + * Deterministic object key for one commit's diff, scoped under the + * author's own eName so keys from different people never collide. + */ + static buildKey( + eName: string, + repoFullName: string, + commitId: string, + ): string { + const owner = eName.replace(/^@/, "").replace(/[^\w.-]/g, "_"); + const repo = repoFullName.replace(/[^\w.-]/g, "_"); + return `diffs/${owner}/${repo}/${commitId}.diff`; + } + + /** + * Uploads a diff's raw text and returns its URL. + * + * `isPublic` mirrors the same repo-visibility signal `deriveAcl` (see + * evault/acl.ts) uses for the MetaEnvelope's own ACL: a private repo's + * diff must not be uploaded `public-read`, or the S3 object itself + * becomes readable by anyone with the URL regardless of what ACL the + * eVault envelope carries - the same protection would be undermined one + * layer down. A private-repo diff is uploaded with no public ACL, so its + * URL is not fetchable without the bucket's own credentials; there is + * deliberately no presigned-URL-on-read feature built here, out of scope + * for this pass. + */ + async uploadDiff( + eName: string, + repoFullName: string, + commitId: string, + diffText: string, + isPublic: boolean, + ): Promise { + const key = S3Storage.buildKey(eName, repoFullName, commitId); + await this.client.send( + new PutObjectCommand({ + Bucket: this.bucket, + Key: key, + Body: Buffer.from(diffText, "utf8"), + ContentType: "text/x-diff", + ...(isPublic ? { ACL: "public-read" as const } : {}), + }), + ); + return `${this.cdnBaseUrl}/${key}`; + } + + /** + * Deterministic object key for a repo's full snapshot - one per repo, not + * per commit or per push, since the whole point is that this object gets + * overwritten in place on every push rather than accumulating one archive + * per commit the way diffs do. See uploadRepoArchive. + */ + static buildArchiveKey(repoFullName: string): string { + const [owner = "", repo = ""] = repoFullName.split("/"); + const safeOwner = owner.replace(/[^\w.-]/g, "_"); + const safeRepo = repo.replace(/[^\w.-]/g, "_"); + return `repos/${safeOwner}/${safeRepo}.zip`; + } + + /** + * Uploads a full repo archive (zip, from GitW3's archive endpoint - see + * content/archive.ts) and returns its URL. The same key is reused on every + * push for a given repo - this call overwrites whatever was there before, + * which is exactly the "replaces whenever anyone makes a commit" behaviour + * the owner-eVault snapshot is meant to have. `isPublic` mirrors the same + * repo-visibility signal `uploadDiff` uses, for the same reason: a private + * repo's full source must not become world-readable via a guessable S3 URL + * just because it's stored this way instead of as a diff. As with + * `uploadDiff`, there is no retroactive re-ACL if the repo's visibility + * changes after this upload - see the spec's Trust model for the + * equivalent, already-accepted limitation on the diff path. + */ + async uploadRepoArchive( + repoFullName: string, + archiveBytes: Buffer, + isPublic: boolean, + ): Promise { + const key = S3Storage.buildArchiveKey(repoFullName); + await this.client.send( + new PutObjectCommand({ + Bucket: this.bucket, + Key: key, + Body: archiveBytes, + ContentType: "application/zip", + ...(isPublic ? { ACL: "public-read" as const } : {}), + }), + ); + return `${this.cdnBaseUrl}/${key}`; + } +} diff --git a/services/forgejo-code-sync/src/sync.test.ts b/services/forgejo-code-sync/src/sync.test.ts new file mode 100644 index 000000000..209749b5c --- /dev/null +++ b/services/forgejo-code-sync/src/sync.test.ts @@ -0,0 +1,195 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { CommitEnvelopePayload, EVaultClient } from "./evault/client.js"; +import type { IdentityResolver } from "./identity.js"; +import { Queue } from "./queue.js"; +import { type DrainDeps, processTask } from "./sync.js"; +import type { CommitSyncTask } from "./task.js"; + +let dir: string; +let queue: Queue; + +const baseTask: CommitSyncTask = { + commitId: "abc123", + repoFullName: "alice/repo", + repoPrivate: false, + ref: "refs/heads/main", + pusherLogin: "alice", + message: "a commit", + committedAt: "2026-08-14T10:00:00Z", + added: ["a.ts"], + removed: [], + modified: [], + commitUrl: "https://git.example.org/alice/repo/commit/abc123", + compareUrl: "https://git.example.org/alice/repo/compare/x...y", +}; + +beforeEach(async () => { + dir = await mkdtemp(path.join(tmpdir(), "forgejo-code-sync-sync-")); + queue = new Queue({ dir }); + await queue.init(); +}); + +afterEach(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +function makeDeps(overrides: Partial = {}): DrainDeps { + const identity = { + resolveEname: vi.fn().mockResolvedValue("@alice"), + } as unknown as IdentityResolver; + + const evault = { + writeCommit: vi.fn().mockResolvedValue("envelope-1"), + } as unknown as EVaultClient; + + const fetchDiff = vi + .fn() + .mockResolvedValue( + "https://s3.example.org/diffs/alice/repo/abc123.diff", + ); + + return { queue, identity, evault, fetchDiff, ...overrides }; +} + +describe("processTask", () => { + it('writes to the eVault with acl ["*"] for a public repo', async () => { + const id = await queue.enqueue(baseTask); + const deps = makeDeps(); + + const outcome = await processTask(id, baseTask, deps); + + expect(outcome).toEqual({ + kind: "succeeded", + envelopeId: "envelope-1", + }); + expect(deps.evault.writeCommit).toHaveBeenCalledWith( + "@alice", + expect.objectContaining({ id: "abc123", authorEName: "@alice" }), + ["*"], + ); + expect(await queue.list()).toEqual([]); + }); + + it("writes to the eVault with an owner-only acl for a private repo", async () => { + const privateTask = { ...baseTask, repoPrivate: true }; + const id = await queue.enqueue(privateTask); + const deps = makeDeps(); + + await processTask(id, privateTask, deps); + + expect(deps.evault.writeCommit).toHaveBeenCalledWith( + "@alice", + expect.anything(), + ["@alice"], + ); + }); + + it("marks an unlinked pusher's task done-and-skipped, never calling writeCommit", async () => { + const id = await queue.enqueue(baseTask); + const deps = makeDeps({ + identity: { + resolveEname: vi.fn().mockResolvedValue(null), + } as unknown as IdentityResolver, + }); + + const outcome = await processTask(id, baseTask, deps); + + expect(outcome).toEqual({ kind: "skipped" }); + expect(deps.evault.writeCommit).not.toHaveBeenCalled(); + // Gone from the queue, same as a success - not lingering as pending/retrying. + expect(await queue.list()).toEqual([]); + }); + + it("leaves a task in the queue for retry on an eVault write failure, rather than dropping it", async () => { + const id = await queue.enqueue(baseTask); + const deps = makeDeps({ + evault: { + writeCommit: vi + .fn() + .mockRejectedValue(new Error("eVault down")), + } as unknown as EVaultClient, + }); + + const outcome = await processTask(id, baseTask, deps); + + expect(outcome.kind).toBe("failed"); + const [task] = await queue.list(); + expect(task).toBeDefined(); + expect(task?.status).toBe("retrying"); + expect(task?.lastError).toBe("eVault down"); + }); + + it("marks the task failed, not skipped, when identity resolution itself errors", async () => { + const id = await queue.enqueue(baseTask); + const deps = makeDeps({ + identity: { + resolveEname: vi + .fn() + .mockRejectedValue( + new Error("GET /users failed: HTTP 500"), + ), + } as unknown as IdentityResolver, + }); + + const outcome = await processTask(id, baseTask, deps); + + expect(outcome.kind).toBe("failed"); + expect(deps.evault.writeCommit).not.toHaveBeenCalled(); + const [task] = await queue.list(); + expect(task?.status).toBe("retrying"); + }); + + it("passes the fetched diffUrl through to the written payload", async () => { + const id = await queue.enqueue(baseTask); + const deps = makeDeps({ + fetchDiff: vi.fn().mockResolvedValue("https://example.org/diff"), + }); + + await processTask(id, baseTask, deps); + + const call = (deps.evault.writeCommit as ReturnType).mock + .calls[0] as [string, CommitEnvelopePayload, string[]]; + expect(call[1].diffUrl).toBe("https://example.org/diff"); + }); + + it("calls fetchDiff with the task and the resolved eName", async () => { + const id = await queue.enqueue(baseTask); + const deps = makeDeps(); + + await processTask(id, baseTask, deps); + + expect(deps.fetchDiff).toHaveBeenCalledWith(baseTask, "@alice"); + }); + + it("leaves a task in the queue for retry when the diff fetch/upload fails", async () => { + const id = await queue.enqueue(baseTask); + const deps = makeDeps({ + fetchDiff: vi.fn().mockRejectedValue(new Error("S3 unreachable")), + }); + + const outcome = await processTask(id, baseTask, deps); + + expect(outcome.kind).toBe("failed"); + expect(deps.evault.writeCommit).not.toHaveBeenCalled(); + const [task] = await queue.list(); + expect(task?.status).toBe("retrying"); + expect(task?.lastError).toBe("S3 unreachable"); + }); + + it("calls onOutcome for every outcome kind", async () => { + const id = await queue.enqueue(baseTask); + const onOutcome = vi.fn(); + const deps = makeDeps({ onOutcome }); + + await processTask(id, baseTask, deps); + + expect(onOutcome).toHaveBeenCalledWith( + id, + baseTask, + expect.objectContaining({ kind: "succeeded" }), + ); + }); +}); diff --git a/services/forgejo-code-sync/src/sync.ts b/services/forgejo-code-sync/src/sync.ts new file mode 100644 index 000000000..08660c4d4 --- /dev/null +++ b/services/forgejo-code-sync/src/sync.ts @@ -0,0 +1,97 @@ +import { deriveAcl } from "./evault/acl.js"; +import type { CommitEnvelopePayload, EVaultClient } from "./evault/client.js"; +import type { IdentityResolver } from "./identity.js"; +import type { Queue, QueueTaskStatus } from "./queue.js"; +import type { CommitSyncTask } from "./task.js"; + +/** + * Fetches a commit's diff and uploads it to S3, returning the resulting URL. + * Throws on any failure - network, non-2xx, S3 upload - there is no longer a + * degraded fallback to return instead; see content/diff.ts. + */ +export type DiffFetcher = ( + task: CommitSyncTask, + eName: string, +) => Promise; + +export type DrainOutcome = + | { kind: "succeeded"; envelopeId: string } + | { kind: "skipped" } + | { kind: "failed"; status: QueueTaskStatus; error: unknown }; + +export interface DrainDeps { + queue: Queue; + identity: IdentityResolver; + evault: EVaultClient; + fetchDiff: DiffFetcher; + /** Called for every outcome, so the caller can log/alert distinctly per kind - see the spec's "Delivery reliability" section. */ + onOutcome?: ( + taskId: string, + task: CommitSyncTask, + outcome: DrainOutcome, + ) => void; +} + +/** + * Processes one queued commit-sync task through to completion: resolve the + * pusher's eName, derive the ACL, fetch the diff, write the MetaEnvelope. + * + * An eName that resolves to `null` (no linked eVault) is a skip, not a failure + * - it must never enter the retry path, and must be indistinguishable from + * neither "still pending" nor "exhausted" from the queue's point of view (see + * queue.ts's `markSkipped`). Any thrown error - a failed identity lookup, an + * eVault write failure - marks the task failed and lets the queue's own + * backoff decide whether to retry it. + */ +export async function processTask( + taskId: string, + task: CommitSyncTask, + deps: DrainDeps, +): Promise { + try { + const eName = await deps.identity.resolveEname(task.pusherLogin); + if (!eName) { + await deps.queue.markSkipped(taskId); + const outcome: DrainOutcome = { kind: "skipped" }; + deps.onOutcome?.(taskId, task, outcome); + return outcome; + } + + const acl = deriveAcl(task.repoPrivate, eName); + const diffUrl = await deps.fetchDiff(task, eName); + + const payload: CommitEnvelopePayload = { + id: task.commitId, + repo: task.repoFullName, + ref: task.ref, + message: task.message, + authorEName: eName, + committedAt: task.committedAt, + added: task.added, + removed: task.removed, + modified: task.modified, + diffUrl, + }; + + const envelopeId = await deps.evault.writeCommit(eName, payload, acl); + await deps.queue.markSucceeded(taskId); + const outcome: DrainOutcome = { kind: "succeeded", envelopeId }; + deps.onOutcome?.(taskId, task, outcome); + return outcome; + } catch (error) { + const status = await deps.queue.markFailed(taskId, error); + const outcome: DrainOutcome = { kind: "failed", status, error }; + deps.onOutcome?.(taskId, task, outcome); + return outcome; + } +} + +/** Drains every currently-due task once. Call on an interval from index.ts. */ +export async function drainOnce(deps: DrainDeps): Promise { + const due = await deps.queue.due(); + const outcomes: DrainOutcome[] = []; + for (const task of due) { + outcomes.push(await processTask(task.id, task.payload, deps)); + } + return outcomes; +} diff --git a/services/forgejo-code-sync/src/task.ts b/services/forgejo-code-sync/src/task.ts new file mode 100644 index 000000000..90d5e2460 --- /dev/null +++ b/services/forgejo-code-sync/src/task.ts @@ -0,0 +1,48 @@ +/** + * Everything the queue's drain loop needs to sync one commit, captured at + * webhook-receipt time so the drain loop never has to go back to the original + * webhook payload (which is not itself persisted - only what's needed is). + */ +export interface CommitSyncTask { + commitId: string; + /** "owner/name" */ + repoFullName: string; + repoPrivate: boolean; + ref: string; + /** The pusher's Forgejo username (`pusher.login`) - resolved to an eName in Phase 3. */ + pusherLogin: string; + message: string; + /** ISO 8601, from the commit's own timestamp. */ + committedAt: string; + added: string[]; + removed: string[]; + modified: string[]; + /** This commit's own GitW3 URL - the diffUrl fallback's first choice. */ + commitUrl: string; + /** The push's compare_url - a fallback if commitUrl is ever unavailable. */ + compareUrl: string; +} + +/** + * Everything the snapshot queue's drain loop needs to sync one push's + * complete repo state into the OWNER's eVault, captured once per webhook + * delivery - not once per commit, unlike CommitSyncTask above. A 10-commit + * push produces exactly one of these, built in webhook/push.ts outside the + * per-commit loop, using the push's final state (`payload.after`) rather than + * any individual commit's sha. + */ +export interface RepoSnapshotTask { + /** "owner/name" */ + repoFullName: string; + repoPrivate: boolean; + ref: string; + /** The repo owner's Forgejo username (`repository.owner.login`) - resolved to an eName the same way a pusher's is. */ + ownerLogin: string; + /** + * The push's own `after` field - the sha the ref points at once this push + * lands, i.e. the exact state the archive endpoint should fetch. Not any + * individual commit's own id: a multi-commit push has several of those, + * and only the final one is "the repo's current state". + */ + headCommitId: string; +} diff --git a/services/forgejo-code-sync/src/webhook/push.test.ts b/services/forgejo-code-sync/src/webhook/push.test.ts new file mode 100644 index 000000000..4be34d998 --- /dev/null +++ b/services/forgejo-code-sync/src/webhook/push.test.ts @@ -0,0 +1,284 @@ +import { createHmac } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import type { Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import express from "express"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { Queue } from "../queue.js"; +import type { CommitSyncTask, RepoSnapshotTask } from "../task.js"; +import { createPushHandlers } from "./push.js"; + +const secret = "test-secret"; + +function sign(body: string): string { + return createHmac("sha256", secret).update(Buffer.from(body)).digest("hex"); +} + +let dir: string; +let snapshotDir: string; +let queue: Queue; +let snapshotQueue: Queue; +let server: Server; +let url: string; + +beforeEach(async () => { + dir = await mkdtemp(path.join(tmpdir(), "forgejo-code-sync-push-")); + snapshotDir = await mkdtemp( + path.join(tmpdir(), "forgejo-code-sync-push-snapshots-"), + ); + queue = new Queue({ dir }); + await queue.init(); + snapshotQueue = new Queue({ dir: snapshotDir }); + await snapshotQueue.init(); + + const app = express(); + app.post( + "/webhook", + ...createPushHandlers({ + commitQueue: queue, + snapshotQueue, + webhookSecret: secret, + }), + ); + + server = app.listen(0); + await new Promise((resolve) => server.once("listening", resolve)); + const { port } = server.address() as AddressInfo; + url = `http://127.0.0.1:${port}/webhook`; +}); + +afterEach(async () => { + await new Promise((resolve) => server.close(() => resolve())); + await rm(dir, { recursive: true, force: true }); + await rm(snapshotDir, { recursive: true, force: true }); +}); + +const twoCommitPayload = { + ref: "refs/heads/main", + after: "bbb222", + compare_url: "https://git.example.org/alice/repo/compare/aaa...bbb", + repository: { + full_name: "alice/repo", + private: false, + owner: { login: "alice" }, + }, + pusher: { login: "alice" }, + commits: [ + { + id: "aaa111", + message: "first commit", + url: "https://git.example.org/alice/repo/commit/aaa111", + timestamp: "2026-08-14T10:00:00Z", + added: ["a.ts"], + removed: [], + modified: [], + }, + { + id: "bbb222", + message: "second commit", + url: "https://git.example.org/alice/repo/commit/bbb222", + timestamp: "2026-08-14T10:05:00Z", + added: [], + removed: [], + modified: ["a.ts"], + }, + ], +}; + +describe("POST /webhook", () => { + it("enqueues one commit task per commit and returns 200 for a validly-signed request", async () => { + const body = JSON.stringify(twoCommitPayload); + const res = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Forgejo-Signature": sign(body), + }, + body, + }); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + ok: true, + queued: 2, + snapshotQueued: true, + }); + + const tasks = await queue.list(); + expect(tasks).toHaveLength(2); + const commitIds = tasks.map((t) => t.payload.commitId).sort(); + expect(commitIds).toEqual(["aaa111", "bbb222"]); + + const first = tasks.find((t) => t.payload.commitId === "aaa111"); + expect(first?.payload).toMatchObject({ + repoFullName: "alice/repo", + repoPrivate: false, + ref: "refs/heads/main", + pusherLogin: "alice", + message: "first commit", + added: ["a.ts"], + }); + }); + + it("enqueues exactly ONE snapshot task for a multi-commit push, not one per commit", async () => { + const body = JSON.stringify(twoCommitPayload); + await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Forgejo-Signature": sign(body), + }, + body, + }); + + const snapshotTasks = await snapshotQueue.list(); + expect(snapshotTasks).toHaveLength(1); + expect(snapshotTasks[0]?.payload).toEqual({ + repoFullName: "alice/repo", + repoPrivate: false, + ref: "refs/heads/main", + ownerLogin: "alice", + headCommitId: "bbb222", // payload.after, not either individual commit id picked arbitrarily + }); + }); + + it("rejects a request with no signature header and enqueues nothing on either queue", async () => { + const body = JSON.stringify(twoCommitPayload); + const res = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body, + }); + + expect(res.status).toBe(401); + expect(await queue.list()).toEqual([]); + expect(await snapshotQueue.list()).toEqual([]); + }); + + it("rejects a request with a wrong signature and enqueues nothing on either queue", async () => { + const body = JSON.stringify(twoCommitPayload); + const res = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Forgejo-Signature": sign('{"tampered":true}'), + }, + body, + }); + + expect(res.status).toBe(401); + expect(await queue.list()).toEqual([]); + expect(await snapshotQueue.list()).toEqual([]); + }); + + it("queues no commit tasks and still returns 200 for a delivery with zero commits, but still queues a snapshot task if the ref moved", async () => { + const payload = { ...twoCommitPayload, commits: [] }; + const body = JSON.stringify(payload); + const res = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Forgejo-Signature": sign(body), + }, + body, + }); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + ok: true, + queued: 0, + snapshotQueued: true, + }); + expect(await queue.list()).toEqual([]); + expect(await snapshotQueue.list()).toHaveLength(1); + }); + + it("queues no snapshot task for a branch/tag deletion push (after is the all-zero sha)", async () => { + const payload = { + ...twoCommitPayload, + after: "0000000000000000000000000000000000000000", + commits: [], + }; + const body = JSON.stringify(payload); + const res = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Forgejo-Signature": sign(body), + }, + body, + }); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + ok: true, + queued: 0, + snapshotQueued: false, + }); + expect(await snapshotQueue.list()).toEqual([]); + }); + + it("captures repository.private correctly for a private repo, on both queues", async () => { + const payload = { + ...twoCommitPayload, + repository: { + full_name: "alice/secret-repo", + private: true, + owner: { login: "alice" }, + }, + commits: [twoCommitPayload.commits[0]], + }; + const body = JSON.stringify(payload); + const res = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Forgejo-Signature": sign(body), + }, + body, + }); + + expect(res.status).toBe(200); + const [task] = await queue.list(); + expect(task?.payload.repoPrivate).toBe(true); + expect(task?.payload.repoFullName).toBe("alice/secret-repo"); + + const [snapshotTask] = await snapshotQueue.list(); + expect(snapshotTask?.payload.repoPrivate).toBe(true); + expect(snapshotTask?.payload.repoFullName).toBe("alice/secret-repo"); + }); + + it("resolves the snapshot task's owner from repository.owner.login, not pusher.login", async () => { + const payload = { + ...twoCommitPayload, + repository: { + full_name: "alice/repo", + private: false, + owner: { login: "alice" }, + }, + // A collaborator pushing to someone else's repo - pusher and owner differ. + pusher: { login: "bob-the-collaborator" }, + }; + const body = JSON.stringify(payload); + await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Forgejo-Signature": sign(body), + }, + body, + }); + + const [snapshotTask] = await snapshotQueue.list(); + expect(snapshotTask?.payload.ownerLogin).toBe("alice"); + + const commitTasks = await queue.list(); + expect( + commitTasks.every( + (t) => t.payload.pusherLogin === "bob-the-collaborator", + ), + ).toBe(true); + }); +}); diff --git a/services/forgejo-code-sync/src/webhook/push.ts b/services/forgejo-code-sync/src/webhook/push.ts new file mode 100644 index 000000000..380a44ea0 --- /dev/null +++ b/services/forgejo-code-sync/src/webhook/push.ts @@ -0,0 +1,115 @@ +import express, { type RequestHandler } from "express"; +import type { Queue } from "../queue.js"; +import type { CommitSyncTask, RepoSnapshotTask } from "../task.js"; +import type { ForgejoPushPayload } from "./pushPayload.js"; +import { verifyForgejoSignature } from "./signature.js"; + +/** Git's all-zero sha, sent as `after` on a branch/tag deletion push - nothing to snapshot. */ +const ALL_ZERO_SHA = /^0+$/; + +export interface PushHandlerDeps { + commitQueue: Queue; + /** A second, independent persisted queue - see queue.ts's own generality and index.ts's wiring. */ + snapshotQueue: Queue; + webhookSecret: string; +} + +/** + * `POST /webhook` - Forgejo's push event. + * + * Route-scoped `express.raw()` rather than a global JSON body parser: the + * signature must be checked against the exact bytes Forgejo sent on the wire + * (`services/webhook/shared/payloader.go`'s `AddDefaultHeaders` signs the raw + * body), and a global `express.json()` would hand this handler an + * already-parsed, already-reserialized object with no guarantee its + * `JSON.stringify` output matches the original bytes. Parsing manually, after + * verifying, sidesteps that entirely rather than relying on a `verify` callback + * elsewhere in the app to capture the raw buffer correctly. + * + * Responds once every commit in the delivery is durably queued, not once + * they're processed - see the spec's "Delivery reliability" section. This + * response is deliberately decoupled from whether the eventual eVault write + * succeeds; that happens later, in the queue's drain loop. + * + * Two independent things get queued per delivery, deliberately at different + * granularity: one CommitSyncTask per commit (the per-pusher commit+diff + * sync, unchanged), and at most ONE RepoSnapshotTask for the whole delivery + * (the per-owner full-repo sync) - a 10-commit push must not upload the whole + * repo ten times. The snapshot task is built outside the commit loop, using + * the push's own `after` (the sha the ref points at once this push lands), + * not any individual commit's id. + */ +export function createPushHandlers(deps: PushHandlerDeps): RequestHandler[] { + const captureRawBody = express.raw({ + type: "application/json", + limit: "10mb", + }); + + const handler: RequestHandler = async (req, res) => { + const rawBody = req.body as Buffer; + const signature = req.header("X-Forgejo-Signature"); + + if ( + !Buffer.isBuffer(rawBody) || + !verifyForgejoSignature(rawBody, deps.webhookSecret, signature) + ) { + res.status(401).json({ error: "invalid signature" }); + return; + } + + let payload: ForgejoPushPayload; + try { + payload = JSON.parse( + rawBody.toString("utf8"), + ) as ForgejoPushPayload; + } catch { + res.status(400).json({ error: "invalid JSON body" }); + return; + } + + const commits = payload.commits ?? []; + for (const commit of commits) { + const task: CommitSyncTask = { + commitId: commit.id, + repoFullName: payload.repository.full_name, + repoPrivate: payload.repository.private, + ref: payload.ref, + pusherLogin: payload.pusher.login, + message: commit.message, + committedAt: commit.timestamp, + added: commit.added, + removed: commit.removed, + modified: commit.modified, + commitUrl: commit.url, + compareUrl: payload.compare_url, + }; + await deps.commitQueue.enqueue(task); + } + + // A branch/tag deletion sends `after` as the all-zero sha - there is + // no ref state left to archive, so no snapshot task is queued for it. + // `payload.after` also being falsy covers older/malformed payloads + // missing the field entirely, the same defensive posture as every + // other field read from this untrusted body. + let snapshotQueued = false; + if (payload.after && !ALL_ZERO_SHA.test(payload.after)) { + const snapshotTask: RepoSnapshotTask = { + repoFullName: payload.repository.full_name, + repoPrivate: payload.repository.private, + ref: payload.ref, + ownerLogin: payload.repository.owner.login, + headCommitId: payload.after, + }; + await deps.snapshotQueue.enqueue(snapshotTask); + snapshotQueued = true; + } + + res.status(200).json({ + ok: true, + queued: commits.length, + snapshotQueued, + }); + }; + + return [captureRawBody, handler]; +} diff --git a/services/forgejo-code-sync/src/webhook/pushPayload.ts b/services/forgejo-code-sync/src/webhook/pushPayload.ts new file mode 100644 index 000000000..efbd82452 --- /dev/null +++ b/services/forgejo-code-sync/src/webhook/pushPayload.ts @@ -0,0 +1,58 @@ +/** + * The shape of a Forgejo `push` event webhook body, restricted to the fields + * this service reads. Field names confirmed against GitW3's own source + * (`modules/structs/hook.go`, `modules/structs/user.go`), not by analogy to + * Gitea/GitHub docs - traps found doing that: + * + * 1. `pusher` is a full `User`, whose username field is JSON-tagged `login`, not + * `username`. GitW3's `services/webhook/notifier.go` builds it via + * `convert.ToUser(ctx, pusher, nil)` with a nil `doer`, so `login_name` is + * never populated here regardless of how the pusher authenticated - resolving + * an eName from this payload alone is not possible, see identity.ts. + * 2. Each commit's `author`/`committer` is a *different* struct (`PayloadUser`), + * whose username field IS JSON-tagged `username`. Free-text git config, never + * validated against any Forgejo account - this service must never resolve + * identity from it. See docs/superpowers/specs/2026-08-14-forgejo-code-sync-design.md + * ("pusher and each commit's author are different things"). + * 3. `repository.owner` is also a full `User` (`modules/structs/repo.go`'s + * `Repository.Owner *User`, `json:"owner"`), and its username field is + * JSON-tagged `login` - the same tag as `pusher.login`, confirmed rather + * than assumed just because both are `*User`. Used for the repo-owner + * snapshot sync - see identity.ts's `IdentityResolver`, reused as-is for + * an owner's username the same way it's used for a pusher's. + * 4. The push's own `after` (`modules/structs/hook.go`'s `PushPayload.After`, + * `json:"after"`) is the sha the ref points at once this push lands - the + * once-per-push repo snapshot uses this, not any individual commit's own + * id, since a multi-commit push has several of those and only the final + * one is "the repo's current state" to archive. + */ +export interface ForgejoPushPayload { + ref: string; + /** The sha this push's ref now points at - the repo-snapshot archive's own ref parameter. */ + after: string; + compare_url: string; + commits: ForgejoPushCommit[]; + repository: { + /** "owner/name", already combined - no need to build it from parts. */ + full_name: string; + private: boolean; + owner: { + /** The repo's owning account. JSON key is "login", same tag as pusher.login - see note 3 above. */ + login: string; + }; + }; + pusher: { + /** The authenticated Forgejo account that ran `git push`. JSON key is "login", not "username". */ + login: string; + }; +} + +export interface ForgejoPushCommit { + id: string; + message: string; + url: string; + timestamp: string; + added: string[]; + removed: string[]; + modified: string[]; +} diff --git a/services/forgejo-code-sync/src/webhook/signature.test.ts b/services/forgejo-code-sync/src/webhook/signature.test.ts new file mode 100644 index 000000000..0396048aa --- /dev/null +++ b/services/forgejo-code-sync/src/webhook/signature.test.ts @@ -0,0 +1,47 @@ +import { createHmac } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { verifyForgejoSignature } from "./signature.js"; + +const secret = "test-secret"; +const body = Buffer.from(JSON.stringify({ ref: "refs/heads/main" })); +const validSignature = createHmac("sha256", secret).update(body).digest("hex"); + +describe("verifyForgejoSignature", () => { + it("accepts a valid, unprefixed signature", () => { + expect(verifyForgejoSignature(body, secret, validSignature)).toBe(true); + }); + + it("rejects a signature computed over a different body (one byte flipped)", () => { + const tamperedBody = Buffer.from( + JSON.stringify({ ref: "refs/heads/mein" }), + ); + expect( + verifyForgejoSignature(tamperedBody, secret, validSignature), + ).toBe(false); + }); + + it("rejects a sha256=-prefixed value - the GitHub-boilerplate trap", () => { + // Regression guard: adapting GitHub-webhook-verification code that + // strips a "sha256=" prefix, then pointing it at X-Forgejo-Signature + // (which carries no prefix), must not accidentally validate. + expect( + verifyForgejoSignature(body, secret, `sha256=${validSignature}`), + ).toBe(false); + }); + + it("rejects a missing header without throwing", () => { + expect(verifyForgejoSignature(body, secret, undefined)).toBe(false); + }); + + it("rejects the wrong secret", () => { + expect( + verifyForgejoSignature(body, "wrong-secret", validSignature), + ).toBe(false); + }); + + it("rejects non-hex garbage without throwing", () => { + expect(verifyForgejoSignature(body, secret, "not-hex-at-all!!")).toBe( + false, + ); + }); +}); diff --git a/services/forgejo-code-sync/src/webhook/signature.ts b/services/forgejo-code-sync/src/webhook/signature.ts new file mode 100644 index 000000000..8c9446d2d --- /dev/null +++ b/services/forgejo-code-sync/src/webhook/signature.ts @@ -0,0 +1,48 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; + +/** + * Verifies a Forgejo webhook's `X-Forgejo-Signature` header. + * + * Two traps, both confirmed against GitW3's own source + * (`services/webhook/shared/payloader.go`'s `AddDefaultHeaders`), not assumed: + * + * 1. The header is the **raw hex digest, with no algorithm prefix**. Forgejo also + * sends a GitHub-compatible `X-Hub-Signature-256` header alongside it, prefixed + * `sha256=` - it is easy to adapt GitHub-webhook-verification boilerplate that + * strips that prefix and point it at this header instead, which silently + * breaks every signature check. This function takes the header's bytes as-is. + * 2. `rawBody` must be the exact bytes Forgejo signed on the wire, not a + * re-serialized `req.body`. If Express's JSON body parser re-`JSON.stringify`s + * the parsed payload before this is called, key ordering or whitespace + * differences make even a correctly-unprefixed comparison fail. Callers must + * capture the raw buffer explicitly (see webhook/push.ts) and pass that here, + * never `JSON.stringify(req.body)`. + * + * `timingSafeEqual`, not `===`, for the same reason as the bridge's + * `client_secret` check - and length is checked first, since `timingSafeEqual` + * throws (rather than returning false) on a length mismatch, which an attacker + * could otherwise use to distinguish "wrong length" from "wrong bytes". + */ +export function verifyForgejoSignature( + rawBody: Buffer, + secret: string, + header: string | undefined, +): boolean { + if (!header) return false; + + const expected = createHmac("sha256", secret).update(rawBody).digest("hex"); + + let received: Buffer; + let expectedBuf: Buffer; + try { + received = Buffer.from(header, "hex"); + expectedBuf = Buffer.from(expected, "hex"); + } catch { + return false; + } + + return ( + received.length === expectedBuf.length && + timingSafeEqual(received, expectedBuf) + ); +} diff --git a/services/forgejo-code-sync/tsconfig.build.json b/services/forgejo-code-sync/tsconfig.build.json new file mode 100644 index 000000000..684176bcb --- /dev/null +++ b/services/forgejo-code-sync/tsconfig.build.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "outDir": "./dist", + "rootDir": "./src", + "declaration": true, + "sourceMap": true + }, + "include": ["src"], + "exclude": ["node_modules", "dist", "**/*.test.ts", "**/*.test-utils.ts"] +} diff --git a/services/forgejo-code-sync/tsconfig.json b/services/forgejo-code-sync/tsconfig.json new file mode 100644 index 000000000..9524c9ba2 --- /dev/null +++ b/services/forgejo-code-sync/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "noUncheckedIndexedAccess": true, + "noEmit": true + }, + "include": ["src", "scripts"], + "exclude": ["node_modules", "dist"] +} diff --git a/services/ontology/schemas/codeCommit.json b/services/ontology/schemas/codeCommit.json new file mode 100644 index 000000000..dc7cbace4 --- /dev/null +++ b/services/ontology/schemas/codeCommit.json @@ -0,0 +1,55 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "schemaId": "af7b8ea0-365c-414b-8dbb-5c0cdd6a46b8", + "title": "CodeCommit", + "type": "object", + "description": "A commit pushed to a GitW3 (Forgejo) repository, synced into the pushing author's own eVault by services/forgejo-code-sync. Identity is resolved from the authenticated pusher, never from the commit's own free-text author/committer fields, which are unverified git config and can name anyone. The diff itself is never inlined here - it is uploaded to S3 and diffUrl points at it, uploaded public-read only when the source repo was public at push time, so the object's own accessibility mirrors this envelope's acl - see docs/superpowers/specs/2026-08-14-forgejo-code-sync-design.md.", + "properties": { + "id": { + "type": "string", + "description": "The commit sha" + }, + "repo": { + "type": "string", + "description": "owner/name of the GitW3 repository" + }, + "ref": { + "type": "string", + "description": "The branch ref the push landed on" + }, + "message": { + "type": "string", + "description": "The commit message" + }, + "authorEName": { + "type": "string", + "description": "eName of the pusher this commit was resolved to. Always the envelope owner's own eName, kept here for consistency with other ontology schemas that carry authorEName even when it's implied by the eVault the envelope lives in" + }, + "committedAt": { + "type": "string", + "format": "date-time", + "description": "The commit's own timestamp, as reported by GitW3" + }, + "added": { + "type": "array", + "items": { "type": "string" }, + "description": "File paths added by this commit" + }, + "removed": { + "type": "array", + "items": { "type": "string" }, + "description": "File paths removed by this commit" + }, + "modified": { + "type": "array", + "items": { "type": "string" }, + "description": "File paths modified by this commit" + }, + "diffUrl": { + "type": "string", + "description": "The commit's unified diff, uploaded to S3 (never inlined - see the schema description). public-read when the source repo was public at push time, otherwise not publicly fetchable" + } + }, + "required": ["id", "repo", "ref", "message", "authorEName", "committedAt", "diffUrl"], + "additionalProperties": false +} diff --git a/services/ontology/schemas/repoSnapshot.json b/services/ontology/schemas/repoSnapshot.json new file mode 100644 index 000000000..1352144a2 --- /dev/null +++ b/services/ontology/schemas/repoSnapshot.json @@ -0,0 +1,36 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "schemaId": "a9b56118-ac82-4f4e-9f70-77444c1a8f34", + "title": "RepoSnapshot", + "type": "object", + "description": "The complete current state of a GitW3 (Forgejo) repository - every file and folder, not a diff - synced into the repository OWNER's own eVault by services/forgejo-code-sync. One envelope per repository, updated in place (via updateMetaEnvelope) on every push rather than a new envelope per push - snapshotUrl always points at an archive of the push's final ref state, replacing whatever it pointed at before. Distinct from codeCommit.json, which is per-pusher and per-commit: this schema is per-repo and per-owner, and exists independently of whether any individual pusher has a linked eVault at all. The archive itself is never inlined here - it is uploaded to S3 and snapshotUrl points at it, uploaded public-read only when the repo was public at the time of that push, so the object's own accessibility mirrors this envelope's acl, the same convention codeCommit.json's diffUrl already uses - see docs/superpowers/specs/2026-08-14-forgejo-code-sync-design.md.", + "properties": { + "repo": { + "type": "string", + "description": "owner/name of the GitW3 repository" + }, + "ref": { + "type": "string", + "description": "The branch ref this snapshot was taken from (the push's own ref, e.g. refs/heads/main)" + }, + "headCommitId": { + "type": "string", + "description": "The sha the ref pointed at when this snapshot was taken - the push payload's own \"after\" field, which is the archive's own ref parameter" + }, + "ownerEName": { + "type": "string", + "description": "eName of the repository owner this snapshot was resolved to. Always the envelope owner's own eName, kept here for consistency with other ontology schemas that carry an eName field even when it's implied by the eVault the envelope lives in" + }, + "snapshotUrl": { + "type": "string", + "description": "A zip archive of the repository at headCommitId, uploaded to S3 (never inlined - see the schema description). public-read when the repo was public at push time, otherwise not publicly fetchable. Overwritten in place on every push - the same S3 key is reused, not a new one per push" + }, + "updatedAt": { + "type": "string", + "format": "date-time", + "description": "When this snapshot was last replaced, as recorded by this service - not a field GitW3 itself reports" + } + }, + "required": ["repo", "ref", "headCommitId", "ownerEName", "snapshotUrl", "updatedAt"], + "additionalProperties": false +}