From 3db1a001fada4f2fbed75876b9295de87c23210b Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 21:53:00 +0000 Subject: [PATCH 01/34] Load server instrumentation asynchronously --- scripts/verify-server-observability-bundle.ts | 209 ++++++++++++++++++ src/app/api/health/route.ts | 12 +- src/instrumentation-node.ts | 5 + src/instrumentation.ts | 9 +- tests/health.test.ts | 54 ++++- 5 files changed, 279 insertions(+), 10 deletions(-) diff --git a/scripts/verify-server-observability-bundle.ts b/scripts/verify-server-observability-bundle.ts index ab8eea46..5dd39867 100644 --- a/scripts/verify-server-observability-bundle.ts +++ b/scripts/verify-server-observability-bundle.ts @@ -24,4 +24,213 @@ for (const path of files) { console.log(`Verified ${files.length} Edge artifacts exclude Node-only observability code.`); +const reservation = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch: () => new Response("reserved"), +}); +const port = reservation.port; +await reservation.stop(true); +const bootProbe = crypto.randomUUID(); + +const server = Bun.spawn( + [ + process.execPath, + "node_modules/next/dist/bin/next", + "start", + "--hostname", + "127.0.0.1", + "--port", + String(port), + ], + { + cwd: process.cwd(), + detached: process.platform !== "win32", + env: { + HOME: process.env.HOME ?? "/tmp", + NEXT_TELEMETRY_DISABLED: "1", + NODE_ENV: "production", + PATH: process.env.PATH ?? "/usr/local/bin:/usr/bin:/bin", + POSTIL_BOOT_PROBE: bootProbe, + POSTIL_SKIP_ENV_VALIDATION: "1", + }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }, +); +const MAX_OUTPUT_CHARACTERS = 4_000; +function collectOutput(stream: ReadableStream): { + cancel: () => Promise; + text: Promise; +} { + const reader = stream.getReader(); + return { + cancel: async () => { + await reader.cancel(); + }, + text: (async () => { + const decoder = new TextDecoder(); + let output = ""; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + output = `${output}${decoder.decode(value, { stream: true })}`.slice( + -MAX_OUTPUT_CHARACTERS, + ); + } + return `${output}${decoder.decode()}`.slice(-MAX_OUTPUT_CHARACTERS); + })(), + }; +} +const stdout = collectOutput(server.stdout); +const stderr = collectOutput(server.stderr); + +let healthy = false; +for (let attempt = 0; attempt < 100 && server.exitCode === null; attempt += 1) { + try { + const response = await fetch(`http://127.0.0.1:${port}/api/health`, { + signal: AbortSignal.timeout(500), + }); + if (response.ok) { + const body = (await response.json()) as Record; + healthy = + body.ok === true && + body.service === "web" && + response.headers.get("x-postil-boot-probe") === bootProbe && + server.exitCode === null; + if (healthy) break; + } else { + await response.body?.cancel(); + } + } catch { + // The production server is still starting. + } + await Bun.sleep(100); +} + +if (healthy) await Bun.sleep(100); +const exitedBeforeTeardown = server.exitCode !== null; + +function signalServer(signal: NodeJS.Signals): boolean { + if (process.platform !== "win32") { + try { + process.kill(-server.pid, signal); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; + throw error; + } + } + if (server.exitCode !== null) return false; + server.kill(signal); + return true; +} + +function serverGroupIsRunning(): boolean { + if (process.platform === "win32") return server.exitCode === null; + try { + process.kill(-server.pid, 0); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; + throw error; + } +} + +async function waitForServerGroup(timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (serverGroupIsRunning() && Date.now() < deadline) { + await Bun.sleep(50); + } + return !serverGroupIsRunning(); +} + +async function terminateServer(): Promise { + if (process.platform === "win32") { + if (server.exitCode !== null) return false; + const taskkill = Bun.spawn( + ["taskkill", "/PID", String(server.pid), "/T", "/F"], + { stdin: "ignore", stdout: "ignore", stderr: "ignore" }, + ); + const taskkillExit = await Promise.race([ + taskkill.exited, + Bun.sleep(2_000).then(() => null), + ]); + if (taskkillExit !== 0) { + if (taskkill.exitCode === null) { + taskkill.kill("SIGKILL"); + const taskkillStopped = await Promise.race([ + taskkill.exited.then(() => true), + Bun.sleep(500).then(() => false), + ]); + if (!taskkillStopped) taskkill.unref(); + } + if (server.exitCode === null) { + server.kill("SIGKILL"); + const serverStopped = await Promise.race([ + server.exited.then(() => true), + Bun.sleep(500).then(() => false), + ]); + if (!serverStopped) server.unref(); + } + return false; + } + const serverStopped = await Promise.race([ + server.exited.then(() => true), + Bun.sleep(2_000).then(() => false), + ]); + if (!serverStopped) { + if (server.exitCode === null) server.kill("SIGKILL"); + const serverReaped = await Promise.race([ + server.exited.then(() => true), + Bun.sleep(500).then(() => false), + ]); + if (!serverReaped) server.unref(); + } + return serverStopped; + } + + if (!serverGroupIsRunning() || !signalServer("SIGTERM")) return false; + const stopped = await waitForServerGroup(2_000); + if (stopped) return true; + + if (!signalServer("SIGKILL")) return false; + const killed = await waitForServerGroup(2_000); + if (!killed) throw new Error("Production server did not stop after SIGKILL."); + return true; +} + +const terminatedByProbe = await terminateServer(); +let outputTimeout: ReturnType | undefined; +const outputTimeoutPromise = new Promise((_, reject) => { + outputTimeout = setTimeout(() => { + reject(new Error("Production server output pipes did not close after teardown.")); + void Promise.allSettled([stdout.cancel(), stderr.cancel()]); + }, 2_000); +}); +let output: string; +try { + output = await Promise.race([ + Promise.all([stdout.text, stderr.text]).then(([out, err]) => + `${out}\n${err}`.trim(), + ), + outputTimeoutPromise, + ]); +} finally { + clearTimeout(outputTimeout); +} +if ( + !healthy || + exitedBeforeTeardown || + !terminatedByProbe || + output.includes("instrumentation hook") +) { + throw new Error( + `Production server failed its boot probe.${output ? `\n${output}` : ""}`, + ); +} + +console.log("Verified the production server loads instrumentation and serves health."); + export {}; diff --git a/src/app/api/health/route.ts b/src/app/api/health/route.ts index e886e46e..8d27d50a 100644 --- a/src/app/api/health/route.ts +++ b/src/app/api/health/route.ts @@ -4,5 +4,15 @@ export const runtime = "nodejs"; export const dynamic = "force-dynamic"; export async function GET(): Promise { - return NextResponse.json({ ok: true, service: "web" }); + const configuredBootProbe = process.env.POSTIL_BOOT_PROBE; + const bootProbe = + process.env.POSTIL_SKIP_ENV_VALIDATION === "1" && + configuredBootProbe && + process.env.POSTIL_BOOT_PROBE_READY === configuredBootProbe + ? configuredBootProbe + : undefined; + return NextResponse.json( + { ok: true, service: "web" }, + bootProbe ? { headers: { "x-postil-boot-probe": bootProbe } } : undefined, + ); } diff --git a/src/instrumentation-node.ts b/src/instrumentation-node.ts index 9045bde0..c90e5f47 100644 --- a/src/instrumentation-node.ts +++ b/src/instrumentation-node.ts @@ -2,6 +2,11 @@ import { validateEnv } from "@/lib/env"; import { reportOperationalFailure } from "@/lib/server-observability"; export function registerNodeInstrumentation(): void { + if (process.env.POSTIL_SKIP_ENV_VALIDATION === "1") { + const bootProbe = process.env.POSTIL_BOOT_PROBE; + if (bootProbe) process.env.POSTIL_BOOT_PROBE_READY = bootProbe; + return; + } validateEnv("web"); } diff --git a/src/instrumentation.ts b/src/instrumentation.ts index 5d7fa2ee..9a9a5945 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -6,16 +6,15 @@ import type { Instrumentation } from "next"; * of failing later on the first request. Skipped during `next build`, which * must not require a live environment. */ -export function register(): void { +export async function register(): Promise { if (process.env.NEXT_PHASE === "phase-production-build") return; if (process.env.NEXT_RUNTIME !== "nodejs") return; - if (process.env.POSTIL_SKIP_ENV_VALIDATION === "1") return; - const { registerNodeInstrumentation } = require("./instrumentation-node") as typeof import("./instrumentation-node"); + const { registerNodeInstrumentation } = await import("./instrumentation-node"); registerNodeInstrumentation(); } -export const onRequestError: Instrumentation.onRequestError = (error) => { +export const onRequestError: Instrumentation.onRequestError = async (error) => { if (process.env.NEXT_RUNTIME !== "nodejs") return; - const { reportNodeRequestError } = require("./instrumentation-node") as typeof import("./instrumentation-node"); + const { reportNodeRequestError } = await import("./instrumentation-node"); reportNodeRequestError(error); }; diff --git a/tests/health.test.ts b/tests/health.test.ts index ef0a6c86..7264fe72 100644 --- a/tests/health.test.ts +++ b/tests/health.test.ts @@ -6,6 +6,8 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { parse } from "yaml"; +import { registerNodeInstrumentation } from "@/instrumentation-node"; + let queryCount = 0; let queryImpl: (text: string) => Promise; @@ -29,13 +31,52 @@ beforeEach(() => { describe("/api/health", () => { test("is cheap process liveness and does not need database configuration", async () => { + const previousBootProbe = process.env.POSTIL_BOOT_PROBE; + const previousDatabaseUrl = process.env.DATABASE_URL; delete process.env.DATABASE_URL; + delete process.env.POSTIL_BOOT_PROBE; - const response = await livenessRoute.GET(); + try { + const response = await livenessRoute.GET(); - expect(response.status).toBe(200); - expect(await response.json()).toEqual({ ok: true, service: "web" }); - expect(queryCount).toBe(0); + expect(response.status).toBe(200); + expect(response.headers.has("x-postil-boot-probe")).toBe(false); + expect(await response.json()).toEqual({ ok: true, service: "web" }); + expect(queryCount).toBe(0); + } finally { + restoreEnvironmentVariable("DATABASE_URL", previousDatabaseUrl); + restoreEnvironmentVariable("POSTIL_BOOT_PROBE", previousBootProbe); + } + }); + + test("echoes the build boot-probe nonce only after instrumentation registers", async () => { + const previousBootProbe = process.env.POSTIL_BOOT_PROBE; + const previousBootProbeReady = process.env.POSTIL_BOOT_PROBE_READY; + const previousSkipValidation = process.env.POSTIL_SKIP_ENV_VALIDATION; + delete process.env.POSTIL_BOOT_PROBE_READY; + delete process.env.POSTIL_SKIP_ENV_VALIDATION; + process.env.POSTIL_BOOT_PROBE = "probe-123"; + try { + const regularResponse = await livenessRoute.GET(); + expect(regularResponse.headers.has("x-postil-boot-probe")).toBe(false); + + process.env.POSTIL_SKIP_ENV_VALIDATION = "1"; + const unregisteredResponse = await livenessRoute.GET(); + expect(unregisteredResponse.headers.has("x-postil-boot-probe")).toBe(false); + + registerNodeInstrumentation(); + const response = await livenessRoute.GET(); + + expect(response.headers.get("x-postil-boot-probe")).toBe("probe-123"); + expect(await response.json()).toEqual({ ok: true, service: "web" }); + } finally { + restoreEnvironmentVariable("POSTIL_BOOT_PROBE", previousBootProbe); + restoreEnvironmentVariable("POSTIL_BOOT_PROBE_READY", previousBootProbeReady); + restoreEnvironmentVariable( + "POSTIL_SKIP_ENV_VALIDATION", + previousSkipValidation, + ); + } }); test("does not import the database module", async () => { @@ -49,6 +90,11 @@ describe("/api/health", () => { }); }); +function restoreEnvironmentVariable(name: string, value: string | undefined): void { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; +} + describe("/api/health/dependencies", () => { test("returns 200 when the database probe succeeds", async () => { queryImpl = async (text: string) => { From a7b9421b2310e0705dc12e69d8ec1d33113cff5c Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 22:10:26 +0000 Subject: [PATCH 02/34] Exercise startup validation in boot probe --- scripts/verify-server-observability-bundle.ts | 9 ++++++++- src/app/api/health/route.ts | 1 - src/instrumentation-node.ts | 8 +++----- tests/health.test.ts | 16 ++-------------- 4 files changed, 13 insertions(+), 21 deletions(-) diff --git a/scripts/verify-server-observability-bundle.ts b/scripts/verify-server-observability-bundle.ts index 5dd39867..a2ab3d1f 100644 --- a/scripts/verify-server-observability-bundle.ts +++ b/scripts/verify-server-observability-bundle.ts @@ -47,12 +47,19 @@ const server = Bun.spawn( cwd: process.cwd(), detached: process.platform !== "win32", env: { + DATABASE_URL: "postgres://postil:postil@127.0.0.1:5432/postil", + GITHUB_OAUTH_CLIENT_ID: "build-probe-client", + GITHUB_OAUTH_CLIENT_SECRET: "build-probe-oauth-secret", + GITHUB_WEBHOOK_SECRET: "build-probe-webhook-secret-32-bytes", HOME: process.env.HOME ?? "/tmp", NEXT_TELEMETRY_DISABLED: "1", NODE_ENV: "production", PATH: process.env.PATH ?? "/usr/local/bin:/usr/bin:/bin", POSTIL_BOOT_PROBE: bootProbe, - POSTIL_SKIP_ENV_VALIDATION: "1", + POSTIL_PUBLIC_URL: "https://postil.invalid", + POSTIL_SEALING_KEY: "00".repeat(32), + POSTIL_SESSION_SECRET: "build-probe-session-secret-32-bytes", + POSTIL_WEBHOOK_DRAIN_ENABLED: "0", }, stdin: "ignore", stdout: "pipe", diff --git a/src/app/api/health/route.ts b/src/app/api/health/route.ts index 8d27d50a..3b4a2a14 100644 --- a/src/app/api/health/route.ts +++ b/src/app/api/health/route.ts @@ -6,7 +6,6 @@ export const dynamic = "force-dynamic"; export async function GET(): Promise { const configuredBootProbe = process.env.POSTIL_BOOT_PROBE; const bootProbe = - process.env.POSTIL_SKIP_ENV_VALIDATION === "1" && configuredBootProbe && process.env.POSTIL_BOOT_PROBE_READY === configuredBootProbe ? configuredBootProbe diff --git a/src/instrumentation-node.ts b/src/instrumentation-node.ts index c90e5f47..8d252c10 100644 --- a/src/instrumentation-node.ts +++ b/src/instrumentation-node.ts @@ -2,12 +2,10 @@ import { validateEnv } from "@/lib/env"; import { reportOperationalFailure } from "@/lib/server-observability"; export function registerNodeInstrumentation(): void { - if (process.env.POSTIL_SKIP_ENV_VALIDATION === "1") { - const bootProbe = process.env.POSTIL_BOOT_PROBE; - if (bootProbe) process.env.POSTIL_BOOT_PROBE_READY = bootProbe; - return; - } + if (process.env.POSTIL_SKIP_ENV_VALIDATION === "1") return; validateEnv("web"); + const bootProbe = process.env.POSTIL_BOOT_PROBE; + if (bootProbe) process.env.POSTIL_BOOT_PROBE_READY = bootProbe; } export function reportNodeRequestError(error: unknown): void { diff --git a/tests/health.test.ts b/tests/health.test.ts index 7264fe72..347f5178 100644 --- a/tests/health.test.ts +++ b/tests/health.test.ts @@ -6,8 +6,6 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { parse } from "yaml"; -import { registerNodeInstrumentation } from "@/instrumentation-node"; - let queryCount = 0; let queryImpl: (text: string) => Promise; @@ -49,22 +47,16 @@ describe("/api/health", () => { } }); - test("echoes the build boot-probe nonce only after instrumentation registers", async () => { + test("echoes the build boot-probe nonce only after instrumentation marks readiness", async () => { const previousBootProbe = process.env.POSTIL_BOOT_PROBE; const previousBootProbeReady = process.env.POSTIL_BOOT_PROBE_READY; - const previousSkipValidation = process.env.POSTIL_SKIP_ENV_VALIDATION; delete process.env.POSTIL_BOOT_PROBE_READY; - delete process.env.POSTIL_SKIP_ENV_VALIDATION; process.env.POSTIL_BOOT_PROBE = "probe-123"; try { - const regularResponse = await livenessRoute.GET(); - expect(regularResponse.headers.has("x-postil-boot-probe")).toBe(false); - - process.env.POSTIL_SKIP_ENV_VALIDATION = "1"; const unregisteredResponse = await livenessRoute.GET(); expect(unregisteredResponse.headers.has("x-postil-boot-probe")).toBe(false); - registerNodeInstrumentation(); + process.env.POSTIL_BOOT_PROBE_READY = "probe-123"; const response = await livenessRoute.GET(); expect(response.headers.get("x-postil-boot-probe")).toBe("probe-123"); @@ -72,10 +64,6 @@ describe("/api/health", () => { } finally { restoreEnvironmentVariable("POSTIL_BOOT_PROBE", previousBootProbe); restoreEnvironmentVariable("POSTIL_BOOT_PROBE_READY", previousBootProbeReady); - restoreEnvironmentVariable( - "POSTIL_SKIP_ENV_VALIDATION", - previousSkipValidation, - ); } }); From f87afb7fd17c81ec8841f285955c130333761f22 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 22:18:25 +0000 Subject: [PATCH 03/34] Generate ephemeral boot probe credentials --- scripts/verify-server-observability-bundle.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/verify-server-observability-bundle.ts b/scripts/verify-server-observability-bundle.ts index a2ab3d1f..f70b9995 100644 --- a/scripts/verify-server-observability-bundle.ts +++ b/scripts/verify-server-observability-bundle.ts @@ -47,18 +47,18 @@ const server = Bun.spawn( cwd: process.cwd(), detached: process.platform !== "win32", env: { - DATABASE_URL: "postgres://postil:postil@127.0.0.1:5432/postil", + DATABASE_URL: "postgres://127.0.0.1/postil", GITHUB_OAUTH_CLIENT_ID: "build-probe-client", - GITHUB_OAUTH_CLIENT_SECRET: "build-probe-oauth-secret", - GITHUB_WEBHOOK_SECRET: "build-probe-webhook-secret-32-bytes", + GITHUB_OAUTH_CLIENT_SECRET: crypto.randomUUID(), + GITHUB_WEBHOOK_SECRET: crypto.randomUUID(), HOME: process.env.HOME ?? "/tmp", NEXT_TELEMETRY_DISABLED: "1", NODE_ENV: "production", PATH: process.env.PATH ?? "/usr/local/bin:/usr/bin:/bin", POSTIL_BOOT_PROBE: bootProbe, POSTIL_PUBLIC_URL: "https://postil.invalid", - POSTIL_SEALING_KEY: "00".repeat(32), - POSTIL_SESSION_SECRET: "build-probe-session-secret-32-bytes", + POSTIL_SEALING_KEY: crypto.randomUUID().replaceAll("-", "").repeat(2), + POSTIL_SESSION_SECRET: crypto.randomUUID(), POSTIL_WEBHOOK_DRAIN_ENABLED: "0", }, stdin: "ignore", From 75e872e27cba2f331303ded46756db3519dc2d68 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 22:57:18 +0000 Subject: [PATCH 04/34] Reconcile publication lifecycle under transaction pooling --- .github/workflows/ci.yml | 4 + src/lib/finding-approvals.ts | 51 ++----------- src/lib/github/publication-threads.ts | 45 +++++++++-- src/lib/publication-receipt.ts | 1 + src/lib/release-job-rollout.ts | 78 ++++++------------- tests/private-worker-gates.test.ts | 42 +++++++++++ tests/publication-receipt.test.ts | 105 +++++++++++++++++++++++--- 7 files changed, 210 insertions(+), 116 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e35590c6..21dcc8e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,6 +71,10 @@ jobs: - run: bun test tests/watchdog.test.ts env: POSTIL_TEST_DATABASE_URL: postgresql://postgres@localhost:5432/postgres + - name: Verify publication lifecycle locking on fresh Postgres + run: bun test --isolate tests/publication-receipt-migration.test.ts + env: + POSTIL_TEST_DATABASE_URL: postgresql://postgres@localhost:5432/postgres - name: Verify self-service billing on fresh Postgres run: | createdb postil_self_service_billing diff --git a/src/lib/finding-approvals.ts b/src/lib/finding-approvals.ts index 11eedb66..eb14ce88 100644 --- a/src/lib/finding-approvals.ts +++ b/src/lib/finding-approvals.ts @@ -315,51 +315,16 @@ export async function withReviewDecisionScopeLock( ): Promise { const client = await pool.connect(); const db = drizzle(client, { schema }); - let pullRequestLocked = false; - let reviewLocked = false; - let identity: string | undefined; try { - const review = ( - await db - .select({ - githubRepoId: schema.repositories.githubRepoId, - prNumber: schema.reviews.prNumber, - }) - .from(schema.reviews) - .innerJoin( - schema.repositories, - eq(schema.repositories.id, schema.reviews.repositoryId), - ) - .where(eq(schema.reviews.id, reviewId)) - .limit(1) - )[0]; - if (!review?.githubRepoId) { - throw new Error("review decision scope is unavailable"); - } - identity = reviewDecisionScopeIdentity(review); - await db.execute( - sql`SELECT pg_advisory_lock(hashtextextended(${`postil:review-pr:${identity}`}, 0))`, - ); - pullRequestLocked = true; - await db.execute(sql`SELECT pg_advisory_lock(${reviewId})`); - reviewLocked = true; - return await operation(db); + return await db.transaction(async (tx) => { + // The production provider transaction-pools connections. Transaction + // advisory locks remain attached to the backend for this bounded + // reconciliation and release automatically on commit or rollback. + await lockReviewDecisionScopeById(tx as Database, reviewId); + return operation(tx as Database); + }); } finally { - try { - if (reviewLocked) { - await db.execute(sql`SELECT pg_advisory_unlock(${reviewId})`); - } - } finally { - try { - if (pullRequestLocked && identity !== undefined) { - await db.execute( - sql`SELECT pg_advisory_unlock(hashtextextended(${`postil:review-pr:${identity}`}, 0))`, - ); - } - } finally { - client.release(); - } - } + client.release(); } } diff --git a/src/lib/github/publication-threads.ts b/src/lib/github/publication-threads.ts index 9e4c2c49..bd3c6603 100644 --- a/src/lib/github/publication-threads.ts +++ b/src/lib/github/publication-threads.ts @@ -8,6 +8,7 @@ interface ThreadNode { id?: string | null; isResolved?: boolean; isOutdated?: boolean; + viewerCanResolve?: boolean; comments?: CommentsConnection | null; } @@ -74,7 +75,10 @@ export async function observeGitHubReviewThreads( const expected = new Set(expectedCommentIds); const observed = new Map< string, - Pick + Pick< + PublicationThreadObservation, + "githubThreadId" | "state" | "viewerCanResolve" + > >(); const timeoutSignal = AbortSignal.timeout(15_000); const requestSignal = signal @@ -102,12 +106,15 @@ export async function observeGitHubReviewThreads( comments: CommentsConnection | null | undefined, githubThreadId: string, state: PublicationThreadObservation["state"], + viewerCanResolve: boolean, ): void { for (const comment of comments?.nodes ?? []) { const id = comment?.databaseId; if (typeof id === "number" && Number.isSafeInteger(id) && id > 0) { const key = String(id); - if (expected.has(key)) observed.set(key, { githubThreadId, state }); + if (expected.has(key)) { + observed.set(key, { githubThreadId, state, viewerCanResolve }); + } } } } @@ -115,6 +122,7 @@ export async function observeGitHubReviewThreads( threadId: string, initialCursor: string, state: PublicationThreadObservation["state"], + viewerCanResolve: boolean, ): Promise { let commentsCursor: string | null = initialCursor; for (let page = 1; page < MAX_PAGES; page += 1) { @@ -138,7 +146,7 @@ export async function observeGitHubReviewThreads( if (!comments) { throw new Error("GitHub review thread comment observation returned no thread"); } - recordComments(comments, threadId, state); + recordComments(comments, threadId, state, viewerCanResolve); if (!comments.pageInfo?.hasNextPage) return; commentsCursor = comments.pageInfo.endCursor ?? null; if (!commentsCursor) { @@ -158,6 +166,7 @@ export async function observeGitHubReviewThreads( id isResolved isOutdated + viewerCanResolve comments(first: ${PAGE_SIZE}) { nodes { databaseId } pageInfo { hasNextPage endCursor } @@ -182,18 +191,31 @@ export async function observeGitHubReviewThreads( if (!thread.id) { throw new Error("GitHub review thread observation omitted its identity"); } + if (typeof thread.viewerCanResolve !== "boolean") { + throw new Error("GitHub review thread observation omitted its resolution capability"); + } const state = thread.isResolved ? "resolved" : thread.isOutdated ? "outdated" : "inline"; - recordComments(thread.comments, thread.id, state); + recordComments( + thread.comments, + thread.id, + state, + thread.viewerCanResolve, + ); if (thread.comments?.pageInfo?.hasNextPage) { const commentsCursor = thread.comments.pageInfo.endCursor ?? null; if (!commentsCursor) { throw new Error("GitHub review thread comment pagination omitted its identity or cursor"); } - await observeRemainingComments(thread.id, commentsCursor, state); + await observeRemainingComments( + thread.id, + commentsCursor, + state, + thread.viewerCanResolve, + ); } } if (!threads.pageInfo?.hasNextPage) { @@ -229,6 +251,19 @@ export async function resolveGitHubReviewThreads( if (!observation.githubThreadId) { throw new Error("GitHub review thread resolution omitted its thread identity"); } + if ( + observation.viewerCanResolve === false && + observation.state === "outdated" + ) { + continue; + } + if (observation.viewerCanResolve !== true) { + throw new Error( + observation.viewerCanResolve === false + ? "GitHub cannot resolve an active Postil review thread" + : "GitHub review thread resolution capability is unknown", + ); + } threadIds.add(observation.githubThreadId); } } diff --git a/src/lib/publication-receipt.ts b/src/lib/publication-receipt.ts index b93bba69..27faa60f 100644 --- a/src/lib/publication-receipt.ts +++ b/src/lib/publication-receipt.ts @@ -386,6 +386,7 @@ export interface PublicationThreadObservation { githubCommentId: string; githubThreadId?: string; state: "inline" | "resolved" | "outdated" | "deleted"; + viewerCanResolve?: boolean; } /** Apply only forge-observed thread state; human prose and review dismissal are not inputs. */ diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index 62796d1b..4bff5fc0 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -51,39 +51,24 @@ export async function publicationLifecycleReleaseActivated( return result.rows[0]?.active === true; } -async function unlockPublicationLifecycleSession( - client: PoolClient, - shared: boolean, -): Promise { - const result = shared - ? await client.query<{ unlocked: boolean }>( - "SELECT pg_advisory_unlock_shared(hashtextextended($1, 0)) AS unlocked", - [PUBLICATION_LIFECYCLE_LOCK], - ) - : await client.query<{ unlocked: boolean }>( - "SELECT pg_advisory_unlock(hashtextextended($1, 0)) AS unlocked", - [PUBLICATION_LIFECYCLE_LOCK], - ); - if (result.rows[0]?.unlocked !== true) { - throw new Error("publication lifecycle session lock was not held"); - } -} - /** Keep gate publication inside the active lifecycle release boundary. */ export async function withPublicationLifecycleReleaseActive( pool: Pool, - operation: (db: Database, client: PoolClient) => Promise, + operation: (db: Database, client: Pool) => Promise, ): Promise { - const client = await pool.connect(); - const db = drizzle(client, { schema }); - let locked = false; + const lockClient = await pool.connect(); + const db = drizzle(pool, { schema }); try { - await client.query( - "SELECT pg_advisory_lock_shared(hashtextextended($1, 0))", + // Transaction pooling can move a client between server sessions after + // each commit. Pin only the capability lock in one bounded transaction; + // the operation uses the pool so its leases and convergence writes remain + // independently visible while deactivation waits on this transaction. + await lockClient.query("BEGIN"); + await lockClient.query( + "SELECT pg_advisory_xact_lock_shared(hashtextextended($1, 0))", [PUBLICATION_LIFECYCLE_LOCK], ); - locked = true; - const active = await client.query<{ active: boolean }>( + const active = await lockClient.query<{ active: boolean }>( `SELECT EXISTS ( SELECT 1 FROM deployment_capabilities WHERE name = $1 ) AS active`, @@ -92,21 +77,14 @@ export async function withPublicationLifecycleReleaseActive( if (active.rows[0]?.active !== true) { throw new PublicationLifecycleReleaseDarkError(); } - return await operation(db, client); + const result = await operation(db, pool); + await lockClient.query("COMMIT"); + return result; + } catch (error) { + await lockClient.query("ROLLBACK").catch(() => undefined); + throw error; } finally { - let releaseError: Error | undefined; - if (locked) { - try { - await unlockPublicationLifecycleSession(client, true); - } catch (error) { - releaseError = - error instanceof Error - ? error - : new Error("publication lifecycle shared lock release failed"); - } - } - client.release(releaseError); - if (releaseError) throw releaseError; + lockClient.release(); } } @@ -161,13 +139,11 @@ export async function activatePublicationLifecycleRelease( released: number; }> { const client = await pool.connect(); - let locked = false; try { - await client.query("SELECT pg_advisory_lock(hashtextextended($1, 0))", [ + await client.query("BEGIN"); + await client.query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [ PUBLICATION_LIFECYCLE_LOCK, ]); - locked = true; - await client.query("BEGIN"); const invalid = await client.query<{ count: string }>( `SELECT count(*)::text AS count FROM reviews AS review @@ -290,19 +266,7 @@ export async function activatePublicationLifecycleRelease( await client.query("ROLLBACK").catch(() => undefined); throw error; } finally { - let releaseError: Error | undefined; - if (locked) { - try { - await unlockPublicationLifecycleSession(client, false); - } catch (error) { - releaseError = - error instanceof Error - ? error - : new Error("publication lifecycle activation lock release failed"); - } - } - client.release(releaseError); - if (releaseError) throw releaseError; + client.release(); } } diff --git a/tests/private-worker-gates.test.ts b/tests/private-worker-gates.test.ts index f8e97412..e9f94195 100644 --- a/tests/private-worker-gates.test.ts +++ b/tests/private-worker-gates.test.ts @@ -175,6 +175,48 @@ describe("private repository worker defense in depth", () => { ); }); + test("publication lifecycle exclusion uses transaction-scoped advisory locks", () => { + const rollout = readFileSync("src/lib/release-job-rollout.ts", "utf8"); + const sharedStart = rollout.indexOf( + "export async function withPublicationLifecycleReleaseActive", + ); + const sharedEnd = rollout.indexOf( + "export async function deactivatePublicationLifecycleRelease", + sharedStart, + ); + const activationStart = rollout.indexOf( + "export async function activatePublicationLifecycleRelease", + ); + const activationEnd = rollout.indexOf( + "function normalizedReleaseSha", + activationStart, + ); + const decisions = readFileSync("src/lib/finding-approvals.ts", "utf8"); + const decisionStart = decisions.indexOf( + "export async function withReviewDecisionScopeLock", + ); + const decisionEnd = decisions.indexOf( + "export async function lockReviewDecisionScopeById", + decisionStart, + ); + + const shared = rollout.slice(sharedStart, sharedEnd); + const activation = rollout.slice(activationStart, activationEnd); + const decision = decisions.slice(decisionStart, decisionEnd); + expect(shared).toContain("pg_advisory_xact_lock_shared"); + expect(shared).toContain("const db = drizzle(pool"); + expect(shared).toContain("operation(db, pool)"); + expect(shared).not.toContain("operation(tx"); + expect(shared).not.toContain("pg_advisory_lock_shared"); + expect(shared).not.toContain("pg_advisory_unlock_shared"); + expect(activation).toContain("pg_advisory_xact_lock"); + expect(activation).not.toContain("pg_advisory_unlock"); + expect(decision).toContain("db.transaction"); + expect(decision).toContain("lockReviewDecisionScopeById"); + expect(decision).not.toContain("pg_advisory_lock("); + expect(decision).not.toContain("pg_advisory_unlock("); + }); + test("respond honors entitlement and release activation before tokens or provider access", () => { const source = readFileSync("src/worker/respond.ts", "utf8"); const start = source.indexOf("export async function runRespondJob"); diff --git a/tests/publication-receipt.test.ts b/tests/publication-receipt.test.ts index 021e43c3..ceae3ebb 100644 --- a/tests/publication-receipt.test.ts +++ b/tests/publication-receipt.test.ts @@ -464,6 +464,7 @@ describe("GitHub publication thread observations", () => { id: "thread-11", isResolved: true, isOutdated: false, + viewerCanResolve: false, comments: { nodes: [{ databaseId: 11 }, { databaseId: 91 }], pageInfo: { hasNextPage: false, endCursor: null }, @@ -473,6 +474,7 @@ describe("GitHub publication thread observations", () => { id: "thread-12", isResolved: false, isOutdated: true, + viewerCanResolve: false, comments: { nodes: [{ databaseId: 12 }], pageInfo: { hasNextPage: false, endCursor: null }, @@ -482,6 +484,7 @@ describe("GitHub publication thread observations", () => { id: "thread-13", isResolved: false, isOutdated: false, + viewerCanResolve: true, comments: { nodes: [{ databaseId: 13 }, { databaseId: 92 }], pageInfo: { hasNextPage: false, endCursor: null }, @@ -500,9 +503,24 @@ describe("GitHub publication thread observations", () => { expect( await observeGitHubReviewThreads("token", "owner/repo", 4, ["11", "12", "13", "14"]), ).toEqual([ - { githubCommentId: "11", githubThreadId: "thread-11", state: "resolved" }, - { githubCommentId: "12", githubThreadId: "thread-12", state: "outdated" }, - { githubCommentId: "13", githubThreadId: "thread-13", state: "inline" }, + { + githubCommentId: "11", + githubThreadId: "thread-11", + state: "resolved", + viewerCanResolve: false, + }, + { + githubCommentId: "12", + githubThreadId: "thread-12", + state: "outdated", + viewerCanResolve: false, + }, + { + githubCommentId: "13", + githubThreadId: "thread-13", + state: "inline", + viewerCanResolve: true, + }, { githubCommentId: "14", state: "deleted" }, ]); }); @@ -522,6 +540,7 @@ describe("GitHub publication thread observations", () => { id: "thread-paged", isResolved: true, isOutdated: false, + viewerCanResolve: false, comments: { nodes: [{ databaseId: 91 }], pageInfo: { hasNextPage: true, endCursor: "comment-page-2" }, @@ -547,7 +566,12 @@ describe("GitHub publication thread observations", () => { }) as unknown as typeof fetch; expect(await observeGitHubReviewThreads("token", "owner/repo", 4, ["11"])).toEqual([ - { githubCommentId: "11", githubThreadId: "thread-paged", state: "resolved" }, + { + githubCommentId: "11", + githubThreadId: "thread-paged", + state: "resolved", + viewerCanResolve: false, + }, ]); expect(requests).toHaveLength(2); expect(requests[1]?.variables).toEqual({ @@ -577,20 +601,79 @@ describe("GitHub publication thread observations", () => { const reconciled = await resolveGitHubReviewThreads( "token", [ - { githubCommentId: "11", githubThreadId: "thread-11", state: "outdated" }, - { githubCommentId: "12", githubThreadId: "thread-12", state: "inline" }, - { githubCommentId: "13", githubThreadId: "thread-13", state: "resolved" }, + { + githubCommentId: "11", + githubThreadId: "thread-11", + state: "outdated", + viewerCanResolve: true, + }, + { + githubCommentId: "12", + githubThreadId: "thread-12", + state: "inline", + viewerCanResolve: true, + }, + { + githubCommentId: "13", + githubThreadId: "thread-13", + state: "resolved", + viewerCanResolve: false, + }, { githubCommentId: "14", state: "deleted" }, + { + githubCommentId: "15", + githubThreadId: "thread-15", + state: "outdated", + viewerCanResolve: false, + }, ], - ["11", "13", "14"], + ["11", "13", "14", "15"], ); expect(requestedThreadIds).toEqual(["thread-11"]); expect(reconciled).toEqual([ - { githubCommentId: "11", githubThreadId: "thread-11", state: "resolved" }, - { githubCommentId: "12", githubThreadId: "thread-12", state: "inline" }, - { githubCommentId: "13", githubThreadId: "thread-13", state: "resolved" }, + { + githubCommentId: "11", + githubThreadId: "thread-11", + state: "resolved", + viewerCanResolve: true, + }, + { + githubCommentId: "12", + githubThreadId: "thread-12", + state: "inline", + viewerCanResolve: true, + }, + { + githubCommentId: "13", + githubThreadId: "thread-13", + state: "resolved", + viewerCanResolve: false, + }, { githubCommentId: "14", state: "deleted" }, + { + githubCommentId: "15", + githubThreadId: "thread-15", + state: "outdated", + viewerCanResolve: false, + }, ]); }); + + test("fails closed when GitHub cannot resolve a still-active terminal thread", async () => { + await expect( + resolveGitHubReviewThreads( + "token", + [ + { + githubCommentId: "16", + githubThreadId: "thread-16", + state: "inline", + viewerCanResolve: false, + }, + ], + ["16"], + ), + ).rejects.toThrow("cannot resolve an active Postil review thread"); + }); }); From 946d04ab6f973c424293645b18004598cde54acd Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 23:06:07 +0000 Subject: [PATCH 05/34] Destroy clients after lifecycle rollback failure --- src/lib/release-job-rollout.ts | 52 ++++++++++++++++++++++++++---- tests/private-worker-gates.test.ts | 3 ++ 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index 4bff5fc0..edb1da85 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -31,6 +31,10 @@ const PUBLICATION_LIFECYCLE_LOCK = "postil:publication-lifecycle-release"; const PUBLICATION_LIFECYCLE_DARK_PAYLOAD_KEY = "_postilPublicationLifecycleDark"; +function databaseClientError(error: unknown, fallback: string): Error { + return error instanceof Error ? error : new Error(fallback); +} + export class PublicationLifecycleReleaseDarkError extends Error { override name = "PublicationLifecycleReleaseDarkError"; @@ -58,6 +62,7 @@ export async function withPublicationLifecycleReleaseActive( ): Promise { const lockClient = await pool.connect(); const db = drizzle(pool, { schema }); + let releaseError: Error | undefined; try { // Transaction pooling can move a client between server sessions after // each commit. Pin only the capability lock in one bounded transaction; @@ -81,10 +86,21 @@ export async function withPublicationLifecycleReleaseActive( await lockClient.query("COMMIT"); return result; } catch (error) { - await lockClient.query("ROLLBACK").catch(() => undefined); + try { + await lockClient.query("ROLLBACK"); + } catch (rollbackError) { + releaseError = databaseClientError( + rollbackError, + "publication lifecycle lock rollback failed", + ); + throw new AggregateError( + [databaseClientError(error, "publication lifecycle operation failed"), releaseError], + "publication lifecycle operation and rollback failed", + ); + } throw error; } finally { - lockClient.release(); + lockClient.release(releaseError); } } @@ -93,6 +109,7 @@ export async function deactivatePublicationLifecycleRelease( pool: Pool, ): Promise<{ deactivated: boolean; parked: number }> { const client = await pool.connect(); + let releaseError: Error | undefined; try { await client.query("BEGIN"); await client.query( @@ -122,10 +139,21 @@ export async function deactivatePublicationLifecycleRelease( parked: parked.rowCount ?? 0, }; } catch (error) { - await client.query("ROLLBACK").catch(() => undefined); + try { + await client.query("ROLLBACK"); + } catch (rollbackError) { + releaseError = databaseClientError( + rollbackError, + "publication lifecycle deactivation rollback failed", + ); + throw new AggregateError( + [databaseClientError(error, "publication lifecycle deactivation failed"), releaseError], + "publication lifecycle deactivation and rollback failed", + ); + } throw error; } finally { - client.release(); + client.release(releaseError); } } @@ -139,6 +167,7 @@ export async function activatePublicationLifecycleRelease( released: number; }> { const client = await pool.connect(); + let releaseError: Error | undefined; try { await client.query("BEGIN"); await client.query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [ @@ -263,10 +292,21 @@ export async function activatePublicationLifecycleRelease( released: released.rowCount ?? 0, }; } catch (error) { - await client.query("ROLLBACK").catch(() => undefined); + try { + await client.query("ROLLBACK"); + } catch (rollbackError) { + releaseError = databaseClientError( + rollbackError, + "publication lifecycle activation rollback failed", + ); + throw new AggregateError( + [databaseClientError(error, "publication lifecycle activation failed"), releaseError], + "publication lifecycle activation and rollback failed", + ); + } throw error; } finally { - client.release(); + client.release(releaseError); } } diff --git a/tests/private-worker-gates.test.ts b/tests/private-worker-gates.test.ts index e9f94195..151afa4a 100644 --- a/tests/private-worker-gates.test.ts +++ b/tests/private-worker-gates.test.ts @@ -207,9 +207,12 @@ describe("private repository worker defense in depth", () => { expect(shared).toContain("const db = drizzle(pool"); expect(shared).toContain("operation(db, pool)"); expect(shared).not.toContain("operation(tx"); + expect(shared).toContain("lockClient.release(releaseError)"); expect(shared).not.toContain("pg_advisory_lock_shared"); expect(shared).not.toContain("pg_advisory_unlock_shared"); expect(activation).toContain("pg_advisory_xact_lock"); + expect(activation).toContain("client.release(releaseError)"); + expect(activation).not.toContain('query("ROLLBACK").catch'); expect(activation).not.toContain("pg_advisory_unlock"); expect(decision).toContain("db.transaction"); expect(decision).toContain("lockReviewDecisionScopeById"); From 2209e90d6256b600618bede70c470d58b1ac994d Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 23:13:46 +0000 Subject: [PATCH 06/34] Pin lifecycle gates to one database transaction --- src/lib/db/index.ts | 46 +++++++++++++- src/lib/finding-approvals.ts | 22 +++---- src/lib/release-job-rollout.ts | 67 +++++++-------------- tests/private-worker-gates.test.ts | 13 ++-- tests/publication-receipt-migration.test.ts | 29 +++++---- 5 files changed, 101 insertions(+), 76 deletions(-) diff --git a/src/lib/db/index.ts b/src/lib/db/index.ts index 566345e6..6258de3e 100644 --- a/src/lib/db/index.ts +++ b/src/lib/db/index.ts @@ -1,5 +1,5 @@ import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres"; -import { Pool } from "pg"; +import { Pool, type PoolClient } from "pg"; import { parse as parseConnectionString } from "pg-connection-string"; import { requireEnv } from "@/lib/env"; @@ -8,6 +8,50 @@ import * as schema from "./schema"; export type Database = NodePgDatabase; +function databaseClientError(error: unknown, fallback: string): Error { + return error instanceof Error ? error : new Error(fallback); +} + +/** + * Run one transaction on a pinned client and discard that client whenever the + * transaction fails. This keeps transaction-scoped locks on one backend and + * prevents an unconfirmed rollback from returning a poisoned client to the + * pool. + */ +export async function withPinnedDatabaseTransaction( + targetPool: Pool, + label: string, + operation: (db: Database, client: PoolClient) => Promise, +): Promise { + const client = await targetPool.connect(); + const clientDatabase = drizzle(client, { schema }); + let bodyError: unknown; + let bodyFailed = false; + let releaseError: Error | undefined; + try { + return await clientDatabase.transaction(async (transaction) => { + try { + return await operation(transaction as Database, client); + } catch (error) { + bodyFailed = true; + bodyError = error; + throw error; + } + }); + } catch (error) { + releaseError = databaseClientError(error, `${label} transaction failed`); + if (bodyFailed && error !== bodyError) { + throw new AggregateError( + [databaseClientError(bodyError, `${label} operation failed`), releaseError], + `${label} operation and transaction cleanup failed`, + ); + } + throw error; + } finally { + client.release(releaseError); + } +} + let pool: Pool | undefined; let database: Database | undefined; diff --git a/src/lib/finding-approvals.ts b/src/lib/finding-approvals.ts index eb14ce88..69d20e7d 100644 --- a/src/lib/finding-approvals.ts +++ b/src/lib/finding-approvals.ts @@ -1,9 +1,7 @@ import { and, desc, eq, isNotNull, isNull, sql } from "drizzle-orm"; -import { drizzle } from "drizzle-orm/node-postgres"; import type { Pool } from "pg"; -import type { Database } from "@/lib/db"; -import { schema } from "@/lib/db"; +import { type Database, schema, withPinnedDatabaseTransaction } from "@/lib/db"; import { computeEffectiveGate, envelopeSchema, @@ -313,19 +311,17 @@ export async function withReviewDecisionScopeLock( reviewId: number, operation: (db: Database) => Promise, ): Promise { - const client = await pool.connect(); - const db = drizzle(client, { schema }); - try { - return await db.transaction(async (tx) => { + return withPinnedDatabaseTransaction( + pool, + "review decision scope", + async (transaction) => { // The production provider transaction-pools connections. Transaction // advisory locks remain attached to the backend for this bounded // reconciliation and release automatically on commit or rollback. - await lockReviewDecisionScopeById(tx as Database, reviewId); - return operation(tx as Database); - }); - } finally { - client.release(); - } + await lockReviewDecisionScopeById(transaction, reviewId); + return operation(transaction); + }, + ); } export async function lockReviewDecisionScopeById( diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index edb1da85..8e149be5 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -1,8 +1,6 @@ -import { drizzle } from "drizzle-orm/node-postgres"; import type { Pool, PoolClient } from "pg"; -import type { Database } from "@/lib/db"; -import * as schema from "@/lib/db/schema"; +import { type Database, withPinnedDatabaseTransaction } from "@/lib/db"; import { OPENROUTER_EXACT_LIMIT_MAX_MICROS } from "@/lib/openrouter-management-adapter"; import { HOSTED_PROVIDER_KEY_LIFECYCLE_JOB_KIND, @@ -58,50 +56,31 @@ export async function publicationLifecycleReleaseActivated( /** Keep gate publication inside the active lifecycle release boundary. */ export async function withPublicationLifecycleReleaseActive( pool: Pool, - operation: (db: Database, client: Pool) => Promise, + operation: (db: Database, client: PoolClient) => Promise, ): Promise { - const lockClient = await pool.connect(); - const db = drizzle(pool, { schema }); - let releaseError: Error | undefined; - try { - // Transaction pooling can move a client between server sessions after - // each commit. Pin only the capability lock in one bounded transaction; - // the operation uses the pool so its leases and convergence writes remain - // independently visible while deactivation waits on this transaction. - await lockClient.query("BEGIN"); - await lockClient.query( - "SELECT pg_advisory_xact_lock_shared(hashtextextended($1, 0))", - [PUBLICATION_LIFECYCLE_LOCK], - ); - const active = await lockClient.query<{ active: boolean }>( - `SELECT EXISTS ( - SELECT 1 FROM deployment_capabilities WHERE name = $1 - ) AS active`, - [PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY], - ); - if (active.rows[0]?.active !== true) { - throw new PublicationLifecycleReleaseDarkError(); - } - const result = await operation(db, pool); - await lockClient.query("COMMIT"); - return result; - } catch (error) { - try { - await lockClient.query("ROLLBACK"); - } catch (rollbackError) { - releaseError = databaseClientError( - rollbackError, - "publication lifecycle lock rollback failed", + return withPinnedDatabaseTransaction( + pool, + "publication lifecycle gate", + async (transaction, client) => { + // Use one transaction for the release lock, leases, nested job staging, + // and convergence writes. Trigger lock requests are then reentrant on + // the same backend even when deactivation is already waiting. + await client.query( + "SELECT pg_advisory_xact_lock_shared(hashtextextended($1, 0))", + [PUBLICATION_LIFECYCLE_LOCK], ); - throw new AggregateError( - [databaseClientError(error, "publication lifecycle operation failed"), releaseError], - "publication lifecycle operation and rollback failed", + const active = await client.query<{ active: boolean }>( + `SELECT EXISTS ( + SELECT 1 FROM deployment_capabilities WHERE name = $1 + ) AS active`, + [PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY], ); - } - throw error; - } finally { - lockClient.release(releaseError); - } + if (active.rows[0]?.active !== true) { + throw new PublicationLifecycleReleaseDarkError(); + } + return operation(transaction, client); + }, + ); } /** Park every gate while a mixed-version fleet can still enqueue old work. */ diff --git a/tests/private-worker-gates.test.ts b/tests/private-worker-gates.test.ts index 151afa4a..795b6184 100644 --- a/tests/private-worker-gates.test.ts +++ b/tests/private-worker-gates.test.ts @@ -192,6 +192,7 @@ describe("private repository worker defense in depth", () => { activationStart, ); const decisions = readFileSync("src/lib/finding-approvals.ts", "utf8"); + const database = readFileSync("src/lib/db/index.ts", "utf8"); const decisionStart = decisions.indexOf( "export async function withReviewDecisionScopeLock", ); @@ -204,20 +205,22 @@ describe("private repository worker defense in depth", () => { const activation = rollout.slice(activationStart, activationEnd); const decision = decisions.slice(decisionStart, decisionEnd); expect(shared).toContain("pg_advisory_xact_lock_shared"); - expect(shared).toContain("const db = drizzle(pool"); - expect(shared).toContain("operation(db, pool)"); - expect(shared).not.toContain("operation(tx"); - expect(shared).toContain("lockClient.release(releaseError)"); + expect(shared).toContain("withPinnedDatabaseTransaction"); + expect(shared).toContain("operation(transaction, client)"); + expect(shared).not.toContain("drizzle(pool"); expect(shared).not.toContain("pg_advisory_lock_shared"); expect(shared).not.toContain("pg_advisory_unlock_shared"); expect(activation).toContain("pg_advisory_xact_lock"); expect(activation).toContain("client.release(releaseError)"); expect(activation).not.toContain('query("ROLLBACK").catch'); expect(activation).not.toContain("pg_advisory_unlock"); - expect(decision).toContain("db.transaction"); + expect(decision).toContain("withPinnedDatabaseTransaction"); expect(decision).toContain("lockReviewDecisionScopeById"); expect(decision).not.toContain("pg_advisory_lock("); expect(decision).not.toContain("pg_advisory_unlock("); + expect(database).toContain("clientDatabase.transaction"); + expect(database).toContain("client.release(releaseError)"); + expect(database).toContain("bodyFailed && error !== bodyError"); }); test("respond honors entitlement and release activation before tokens or provider access", () => { diff --git a/tests/publication-receipt-migration.test.ts b/tests/publication-receipt-migration.test.ts index 6743ff66..ed78bea6 100644 --- a/tests/publication-receipt-migration.test.ts +++ b/tests/publication-receipt-migration.test.ts @@ -612,9 +612,16 @@ describeDb("publication receipt migration and lifecycle", () => { }); const publication = withPublicationLifecycleReleaseActive( pool, - async () => { + async (_lockedDb, lockedClient) => { publicationLocked(); await publicationHold; + await lockedClient.query( + `INSERT INTO jobs (kind, payload) + VALUES ('gate-state-sync', jsonb_build_object( + 'reviewId', $1::bigint, 'reviewPublicId', $2::text + ))`, + [reviewId, review.rows[0]!.public_id], + ); }, ); await publicationAcquired; @@ -630,19 +637,15 @@ describeDb("publication receipt migration and lifecycle", () => { finishPublication(); await publication; await expect(deactivation).resolves.toMatchObject({ deactivated: true }); - const secondGate = await pool.query<{ id: string }>( - `INSERT INTO jobs (kind, payload) - VALUES ('gate-state-sync', jsonb_build_object( - 'reviewId', $1::bigint, 'reviewPublicId', $2::text - )) - RETURNING id`, - [reviewId, review.rows[0]!.public_id], - ); - const parkedAfter = await pool.query<{ parked: boolean }>( - "SELECT run_after = 'infinity'::timestamptz AS parked FROM jobs WHERE id = $1", - [secondGate.rows[0]!.id], + const parkedAfter = await pool.query<{ count: string }>( + `SELECT count(*)::text AS count + FROM jobs + WHERE kind = 'gate-state-sync' + AND payload->>'reviewPublicId' = $1 + AND run_after = 'infinity'::timestamptz`, + [review.rows[0]!.public_id], ); - expect(parkedAfter.rows[0]?.parked).toBe(true); + expect(Number(parkedAfter.rows[0]?.count ?? "0")).toBeGreaterThan(0); expect(await activatePublicationLifecycleRelease(pool)).toMatchObject({ activated: true, }); From fd62624e732805f68c88f3e9920eafc632fea4d2 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 23:18:06 +0000 Subject: [PATCH 07/34] Order gate publisher locks consistently --- src/worker/gate-state-sync.ts | 4 ++++ tests/gate-state-sync-job.test.ts | 11 ++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/worker/gate-state-sync.ts b/src/worker/gate-state-sync.ts index 6270470e..057eb524 100644 --- a/src/worker/gate-state-sync.ts +++ b/src/worker/gate-state-sync.ts @@ -49,6 +49,10 @@ export async function runGateStateSyncJob( } validateReviewPayload(payload); const leaseId = randomUUID(); + // Publication reconciliation takes the review advisory lock before + // updating this row. Keep the same order while the lifecycle wrapper holds + // the outer transaction so the publisher lease cannot invert those locks. + await lockReviewApprovalState(db, payload.reviewId); if (!(await acquireGatePublisherLease(db, payload, leaseId))) return; try { for (let iteration = 0; iteration < 8; iteration += 1) { diff --git a/tests/gate-state-sync-job.test.ts b/tests/gate-state-sync-job.test.ts index a596803e..fa919358 100644 --- a/tests/gate-state-sync-job.test.ts +++ b/tests/gate-state-sync-job.test.ts @@ -19,6 +19,7 @@ let tokenReleaseResolve: (() => void) | null = null; let tokenEntered = Promise.resolve(); let tokenRelease = Promise.resolve(); let loseLeaseAfterCheck = false; +let operationOrder: string[] = []; const row = { id: 7, @@ -78,6 +79,7 @@ function updateChain() { }, returning() { if ("gateSyncLeaseId" in values) { + operationOrder.push("lease"); if (leaseHeld) return Promise.resolve([]); leaseHeld = true; } @@ -163,6 +165,7 @@ mock.module("@/lib/finding-approvals", () => ({ hasNewerCompletedReviewForHead: async () => false, lockReviewApprovalState: async () => { lockCalls += 1; + operationOrder.push("lock"); }, parseEnvelopeForApprovals: () => ({ version: 1 }), updateStoredEffectiveGate: async ( @@ -233,6 +236,7 @@ beforeEach(() => { leaseHeld = false; blockToken = false; loseLeaseAfterCheck = false; + operationOrder = []; row.publicationLifecycleReconciledAt = new Date(); row.publicationLifecycleRequiredAt = new Date(); tokenEntered = new Promise((resolve) => { @@ -266,7 +270,8 @@ describe("durable gate state synchronization", () => { test("recomputes state under an advisory lock before publishing", async () => { await runGateStateSyncJob({ reviewId: 7, reviewPublicId: row.publicId }); - expect(lockCalls).toBe(2); + expect(lockCalls).toBe(3); + expect(operationOrder.slice(0, 2)).toEqual(["lock", "lease"]); expect(storedStates).toEqual([false]); expect(checkCalls).toEqual([ { @@ -293,7 +298,7 @@ describe("durable gate state synchronization", () => { effectiveFailing = true; await runGateStateSyncJob({ reviewId: 7, reviewPublicId: row.publicId }); - expect(lockCalls).toBe(3); + expect(lockCalls).toBe(5); expect(storedStates).toEqual([true]); expect(checkCalls.map((call) => call.conclusion)).toEqual(["success", "failure"]); expect(checkCalls[1]?.detailsUrl).toBe( @@ -382,6 +387,6 @@ describe("durable gate state synchronization", () => { ).rejects.toThrow(); expect(transactionsFinalized).toBe(1); - expect(lockCalls).toBe(1); + expect(lockCalls).toBe(2); }); }); From 4c3fd318b34f07dde7ad43a641598e522437c794 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 23:19:35 +0000 Subject: [PATCH 08/34] Isolate pinned database transactions --- src/lib/db-transaction.ts | 49 ++++++++++++++++++++++++++++++ src/lib/db/index.ts | 46 +--------------------------- src/lib/finding-approvals.ts | 3 +- src/lib/release-job-rollout.ts | 3 +- tests/private-worker-gates.test.ts | 2 +- 5 files changed, 55 insertions(+), 48 deletions(-) create mode 100644 src/lib/db-transaction.ts diff --git a/src/lib/db-transaction.ts b/src/lib/db-transaction.ts new file mode 100644 index 00000000..aac022be --- /dev/null +++ b/src/lib/db-transaction.ts @@ -0,0 +1,49 @@ +import { drizzle } from "drizzle-orm/node-postgres"; +import type { Pool, PoolClient } from "pg"; + +import type { Database } from "@/lib/db"; +import * as schema from "@/lib/db/schema"; + +function databaseClientError(error: unknown, fallback: string): Error { + return error instanceof Error ? error : new Error(fallback); +} + +/** + * Run one transaction on a pinned client and discard that client whenever the + * transaction fails. This keeps transaction-scoped locks on one backend and + * prevents an unconfirmed rollback from returning a poisoned client to the + * pool. + */ +export async function withPinnedDatabaseTransaction( + pool: Pool, + label: string, + operation: (db: Database, client: PoolClient) => Promise, +): Promise { + const client = await pool.connect(); + const clientDatabase = drizzle(client, { schema }); + let bodyError: unknown; + let bodyFailed = false; + let releaseError: Error | undefined; + try { + return await clientDatabase.transaction(async (transaction) => { + try { + return await operation(transaction as Database, client); + } catch (error) { + bodyFailed = true; + bodyError = error; + throw error; + } + }); + } catch (error) { + releaseError = databaseClientError(error, `${label} transaction failed`); + if (bodyFailed && error !== bodyError) { + throw new AggregateError( + [databaseClientError(bodyError, `${label} operation failed`), releaseError], + `${label} operation and transaction cleanup failed`, + ); + } + throw error; + } finally { + client.release(releaseError); + } +} diff --git a/src/lib/db/index.ts b/src/lib/db/index.ts index 6258de3e..566345e6 100644 --- a/src/lib/db/index.ts +++ b/src/lib/db/index.ts @@ -1,5 +1,5 @@ import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres"; -import { Pool, type PoolClient } from "pg"; +import { Pool } from "pg"; import { parse as parseConnectionString } from "pg-connection-string"; import { requireEnv } from "@/lib/env"; @@ -8,50 +8,6 @@ import * as schema from "./schema"; export type Database = NodePgDatabase; -function databaseClientError(error: unknown, fallback: string): Error { - return error instanceof Error ? error : new Error(fallback); -} - -/** - * Run one transaction on a pinned client and discard that client whenever the - * transaction fails. This keeps transaction-scoped locks on one backend and - * prevents an unconfirmed rollback from returning a poisoned client to the - * pool. - */ -export async function withPinnedDatabaseTransaction( - targetPool: Pool, - label: string, - operation: (db: Database, client: PoolClient) => Promise, -): Promise { - const client = await targetPool.connect(); - const clientDatabase = drizzle(client, { schema }); - let bodyError: unknown; - let bodyFailed = false; - let releaseError: Error | undefined; - try { - return await clientDatabase.transaction(async (transaction) => { - try { - return await operation(transaction as Database, client); - } catch (error) { - bodyFailed = true; - bodyError = error; - throw error; - } - }); - } catch (error) { - releaseError = databaseClientError(error, `${label} transaction failed`); - if (bodyFailed && error !== bodyError) { - throw new AggregateError( - [databaseClientError(bodyError, `${label} operation failed`), releaseError], - `${label} operation and transaction cleanup failed`, - ); - } - throw error; - } finally { - client.release(releaseError); - } -} - let pool: Pool | undefined; let database: Database | undefined; diff --git a/src/lib/finding-approvals.ts b/src/lib/finding-approvals.ts index 69d20e7d..36a91f56 100644 --- a/src/lib/finding-approvals.ts +++ b/src/lib/finding-approvals.ts @@ -1,7 +1,8 @@ import { and, desc, eq, isNotNull, isNull, sql } from "drizzle-orm"; import type { Pool } from "pg"; -import { type Database, schema, withPinnedDatabaseTransaction } from "@/lib/db"; +import { type Database, schema } from "@/lib/db"; +import { withPinnedDatabaseTransaction } from "@/lib/db-transaction"; import { computeEffectiveGate, envelopeSchema, diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index 8e149be5..fa72e7b1 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -1,6 +1,7 @@ import type { Pool, PoolClient } from "pg"; -import { type Database, withPinnedDatabaseTransaction } from "@/lib/db"; +import type { Database } from "@/lib/db"; +import { withPinnedDatabaseTransaction } from "@/lib/db-transaction"; import { OPENROUTER_EXACT_LIMIT_MAX_MICROS } from "@/lib/openrouter-management-adapter"; import { HOSTED_PROVIDER_KEY_LIFECYCLE_JOB_KIND, diff --git a/tests/private-worker-gates.test.ts b/tests/private-worker-gates.test.ts index 795b6184..26e69023 100644 --- a/tests/private-worker-gates.test.ts +++ b/tests/private-worker-gates.test.ts @@ -192,7 +192,7 @@ describe("private repository worker defense in depth", () => { activationStart, ); const decisions = readFileSync("src/lib/finding-approvals.ts", "utf8"); - const database = readFileSync("src/lib/db/index.ts", "utf8"); + const database = readFileSync("src/lib/db-transaction.ts", "utf8"); const decisionStart = decisions.indexOf( "export async function withReviewDecisionScopeLock", ); From 85245eb956be8a69c1e74c8d1d41939d8329b1de Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 23:25:04 +0000 Subject: [PATCH 09/34] Order publication lifecycle locks globally --- src/lib/finding-approvals.ts | 2 ++ src/lib/publication-lifecycle-lock.ts | 15 +++++++++++++++ src/lib/release-job-rollout.ts | 10 +++++----- tests/private-worker-gates.test.ts | 11 ++++++++++- 4 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 src/lib/publication-lifecycle-lock.ts diff --git a/src/lib/finding-approvals.ts b/src/lib/finding-approvals.ts index 36a91f56..a627cd4b 100644 --- a/src/lib/finding-approvals.ts +++ b/src/lib/finding-approvals.ts @@ -3,6 +3,7 @@ import type { Pool } from "pg"; import { type Database, schema } from "@/lib/db"; import { withPinnedDatabaseTransaction } from "@/lib/db-transaction"; +import { lockPublicationLifecycleShared } from "@/lib/publication-lifecycle-lock"; import { computeEffectiveGate, envelopeSchema, @@ -319,6 +320,7 @@ export async function withReviewDecisionScopeLock( // The production provider transaction-pools connections. Transaction // advisory locks remain attached to the backend for this bounded // reconciliation and release automatically on commit or rollback. + await lockPublicationLifecycleShared(transaction); await lockReviewDecisionScopeById(transaction, reviewId); return operation(transaction); }, diff --git a/src/lib/publication-lifecycle-lock.ts b/src/lib/publication-lifecycle-lock.ts new file mode 100644 index 00000000..37b167b8 --- /dev/null +++ b/src/lib/publication-lifecycle-lock.ts @@ -0,0 +1,15 @@ +import { sql } from "drizzle-orm"; + +import type { Database } from "@/lib/db"; + +export const PUBLICATION_LIFECYCLE_LOCK = + "postil:publication-lifecycle-release"; + +/** Keep lifecycle work ahead of narrower review locks in the global order. */ +export async function lockPublicationLifecycleShared( + database: Database, +): Promise { + await database.execute( + sql`SELECT pg_advisory_xact_lock_shared(hashtextextended(${PUBLICATION_LIFECYCLE_LOCK}, 0))`, + ); +} diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index fa72e7b1..8d0f64b8 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -2,6 +2,10 @@ import type { Pool, PoolClient } from "pg"; import type { Database } from "@/lib/db"; import { withPinnedDatabaseTransaction } from "@/lib/db-transaction"; +import { + lockPublicationLifecycleShared, + PUBLICATION_LIFECYCLE_LOCK, +} from "@/lib/publication-lifecycle-lock"; import { OPENROUTER_EXACT_LIMIT_MAX_MICROS } from "@/lib/openrouter-management-adapter"; import { HOSTED_PROVIDER_KEY_LIFECYCLE_JOB_KIND, @@ -26,7 +30,6 @@ export const HOSTED_INFERENCE_FLEET_ACTIVE_CAPABILITY = export const HOSTED_INFERENCE_LOCK = "postil:hosted-inference-release"; export const PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY = "publication-lifecycle-fleet-active"; -const PUBLICATION_LIFECYCLE_LOCK = "postil:publication-lifecycle-release"; const PUBLICATION_LIFECYCLE_DARK_PAYLOAD_KEY = "_postilPublicationLifecycleDark"; @@ -66,10 +69,7 @@ export async function withPublicationLifecycleReleaseActive( // Use one transaction for the release lock, leases, nested job staging, // and convergence writes. Trigger lock requests are then reentrant on // the same backend even when deactivation is already waiting. - await client.query( - "SELECT pg_advisory_xact_lock_shared(hashtextextended($1, 0))", - [PUBLICATION_LIFECYCLE_LOCK], - ); + await lockPublicationLifecycleShared(transaction); const active = await client.query<{ active: boolean }>( `SELECT EXISTS ( SELECT 1 FROM deployment_capabilities WHERE name = $1 diff --git a/tests/private-worker-gates.test.ts b/tests/private-worker-gates.test.ts index 26e69023..cab45cbb 100644 --- a/tests/private-worker-gates.test.ts +++ b/tests/private-worker-gates.test.ts @@ -193,6 +193,10 @@ describe("private repository worker defense in depth", () => { ); const decisions = readFileSync("src/lib/finding-approvals.ts", "utf8"); const database = readFileSync("src/lib/db-transaction.ts", "utf8"); + const lifecycleLock = readFileSync( + "src/lib/publication-lifecycle-lock.ts", + "utf8", + ); const decisionStart = decisions.indexOf( "export async function withReviewDecisionScopeLock", ); @@ -204,8 +208,9 @@ describe("private repository worker defense in depth", () => { const shared = rollout.slice(sharedStart, sharedEnd); const activation = rollout.slice(activationStart, activationEnd); const decision = decisions.slice(decisionStart, decisionEnd); - expect(shared).toContain("pg_advisory_xact_lock_shared"); + expect(lifecycleLock).toContain("pg_advisory_xact_lock_shared"); expect(shared).toContain("withPinnedDatabaseTransaction"); + expect(shared).toContain("lockPublicationLifecycleShared(transaction)"); expect(shared).toContain("operation(transaction, client)"); expect(shared).not.toContain("drizzle(pool"); expect(shared).not.toContain("pg_advisory_lock_shared"); @@ -215,7 +220,11 @@ describe("private repository worker defense in depth", () => { expect(activation).not.toContain('query("ROLLBACK").catch'); expect(activation).not.toContain("pg_advisory_unlock"); expect(decision).toContain("withPinnedDatabaseTransaction"); + expect(decision).toContain("lockPublicationLifecycleShared(transaction)"); expect(decision).toContain("lockReviewDecisionScopeById"); + expect(decision.indexOf("lockPublicationLifecycleShared(transaction)")).toBeLessThan( + decision.indexOf("lockReviewDecisionScopeById"), + ); expect(decision).not.toContain("pg_advisory_lock("); expect(decision).not.toContain("pg_advisory_unlock("); expect(database).toContain("clientDatabase.transaction"); From 3fc06caba6230028f6f381241f4ab3a21cabf81d Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 23:31:23 +0000 Subject: [PATCH 10/34] Make lifecycle triggers nonblocking during deploys --- ...ication_lifecycle_nonblocking_triggers.sql | 60 + drizzle/meta/0059_snapshot.json | 7989 +++++++++++++++++ drizzle/meta/_journal.json | 7 + tests/publication-receipt-migration.test.ts | 67 + 4 files changed, 8123 insertions(+) create mode 100644 drizzle/0059_publication_lifecycle_nonblocking_triggers.sql create mode 100644 drizzle/meta/0059_snapshot.json diff --git a/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql b/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql new file mode 100644 index 00000000..c3d2c16f --- /dev/null +++ b/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql @@ -0,0 +1,60 @@ +SELECT pg_advisory_xact_lock(hashtextextended('postil:publication-lifecycle-release', 0));--> statement-breakpoint +CREATE OR REPLACE FUNCTION "postil_require_publication_lifecycle"() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + -- Include ordinary completions in release quiescence when possible, but do + -- not wait behind a queued deactivation while the UPDATE already owns its + -- review row. The lifecycle marker is monotonic and its gate is staged by + -- the companion trigger below. + PERFORM pg_try_advisory_xact_lock_shared( + hashtextextended('postil:publication-lifecycle-release', 0) + ); + IF NEW.publication_lifecycle_required_at IS NULL + AND NEW.envelope IS NOT NULL + AND NEW.status IN ('running', 'completed') + AND ( + TG_OP = 'INSERT' + OR OLD.envelope IS NULL + OR OLD.status NOT IN ('running', 'completed') + OR ( + NEW.status = 'completed' + AND OLD.status IS DISTINCT FROM 'completed' + ) + ) + THEN + NEW.publication_lifecycle_required_at := now(); + END IF; + RETURN NEW; +END; +$$;--> statement-breakpoint +CREATE OR REPLACE FUNCTION "postil_stage_gate_sync_until_publication_lifecycle_activation"() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + lifecycle_active boolean := false; +BEGIN + -- A failed try-lock means deactivation owns or is queued for the boundary. + -- Park the job without waiting while its caller may hold narrower locks. + IF pg_try_advisory_xact_lock_shared( + hashtextextended('postil:publication-lifecycle-release', 0) + ) THEN + SELECT EXISTS ( + SELECT 1 FROM deployment_capabilities + WHERE name = 'publication-lifecycle-fleet-active' + ) INTO lifecycle_active; + END IF; + IF NOT lifecycle_active THEN + NEW.run_after := 'infinity'::timestamptz; + NEW.payload := jsonb_set( + COALESCE(NEW.payload, '{}'::jsonb), + '{_postilPublicationLifecycleDark}', + 'true'::jsonb, + true + ); + END IF; + RETURN NEW; +END; +$$; diff --git a/drizzle/meta/0059_snapshot.json b/drizzle/meta/0059_snapshot.json new file mode 100644 index 00000000..277c249c --- /dev/null +++ b/drizzle/meta/0059_snapshot.json @@ -0,0 +1,7989 @@ +{ + "id": "64f407fd-a354-4c9b-a8a3-f267886fcbc1", + "prevId": "d1c76f4c-f46c-4875-9512-e1069f26013e", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.billing_author_settlements": { + "name": "billing_author_settlements", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "provider_subscription_id": { + "name": "provider_subscription_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "period_starts_at": { + "name": "period_starts_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "period_ends_at": { + "name": "period_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "active_author_count": { + "name": "active_author_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "unit_amount_cents": { + "name": "unit_amount_cents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 600 + }, + "total_amount_cents": { + "name": "total_amount_cents", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "provider_transaction_id": { + "name": "provider_transaction_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "attempt_started_at": { + "name": "attempt_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "next_reconcile_at": { + "name": "next_reconcile_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error_category": { + "name": "last_error_category", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "billing_author_settlements_org_period_idx": { + "name": "billing_author_settlements_org_period_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "period_starts_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "period_ends_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "billing_author_settlements_provider_transaction_idx": { + "name": "billing_author_settlements_provider_transaction_idx", + "columns": [ + { + "expression": "provider_transaction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "billing_author_settlements_status_reconcile_idx": { + "name": "billing_author_settlements_status_reconcile_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_reconcile_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "billing_author_settlements_org_id_organizations_id_fk": { + "name": "billing_author_settlements_org_id_organizations_id_fk", + "tableFrom": "billing_author_settlements", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "restrict" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "billing_author_settlements_period_check": { + "name": "billing_author_settlements_period_check", + "value": "\"billing_author_settlements\".\"period_starts_at\" < \"billing_author_settlements\".\"period_ends_at\"" + }, + "billing_author_settlements_author_count_check": { + "name": "billing_author_settlements_author_count_check", + "value": "\"billing_author_settlements\".\"active_author_count\" >= 0" + }, + "billing_author_settlements_amount_check": { + "name": "billing_author_settlements_amount_check", + "value": "\"billing_author_settlements\".\"unit_amount_cents\" = 600 AND \"billing_author_settlements\".\"total_amount_cents\" = \"billing_author_settlements\".\"active_author_count\" * \"billing_author_settlements\".\"unit_amount_cents\"" + }, + "billing_author_settlements_status_check": { + "name": "billing_author_settlements_status_check", + "value": "\"billing_author_settlements\".\"status\" IN ('pending', 'charging', 'reconciling', 'charged', 'no_charge', 'failed')" + }, + "billing_author_settlements_attempt_count_check": { + "name": "billing_author_settlements_attempt_count_check", + "value": "\"billing_author_settlements\".\"attempt_count\" >= 0" + }, + "billing_author_settlements_subscription_nonempty": { + "name": "billing_author_settlements_subscription_nonempty", + "value": "length(btrim(\"billing_author_settlements\".\"provider_subscription_id\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.billing_checkout_transactions": { + "name": "billing_checkout_transactions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "requested_by_user_id": { + "name": "requested_by_user_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'paddle'" + }, + "provider_transaction_id": { + "name": "provider_transaction_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "checkout_url": { + "name": "checkout_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'creating'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_error_category": { + "name": "last_error_category", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "billing_checkout_transactions_provider_transaction_idx": { + "name": "billing_checkout_transactions_provider_transaction_idx", + "columns": [ + { + "expression": "provider_transaction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "billing_checkout_transactions_open_org_idx": { + "name": "billing_checkout_transactions_open_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "where": "\"billing_checkout_transactions\".\"status\" IN ('creating', 'pending')", + "concurrently": false + }, + "billing_checkout_transactions_status_expiry_idx": { + "name": "billing_checkout_transactions_status_expiry_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "billing_checkout_transactions_org_id_organizations_id_fk": { + "name": "billing_checkout_transactions_org_id_organizations_id_fk", + "tableFrom": "billing_checkout_transactions", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "billing_checkout_transactions_requested_by_user_id_users_id_fk": { + "name": "billing_checkout_transactions_requested_by_user_id_users_id_fk", + "tableFrom": "billing_checkout_transactions", + "columnsFrom": [ + "requested_by_user_id" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "restrict" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "billing_checkout_transactions_provider_check": { + "name": "billing_checkout_transactions_provider_check", + "value": "\"billing_checkout_transactions\".\"provider\" = 'paddle'" + }, + "billing_checkout_transactions_status_check": { + "name": "billing_checkout_transactions_status_check", + "value": "\"billing_checkout_transactions\".\"status\" IN ('creating', 'pending', 'completed', 'failed', 'expired', 'canceled')" + }, + "billing_checkout_transactions_provider_transaction_nonempty": { + "name": "billing_checkout_transactions_provider_transaction_nonempty", + "value": "\"billing_checkout_transactions\".\"provider_transaction_id\" IS NULL OR length(btrim(\"billing_checkout_transactions\".\"provider_transaction_id\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.billing_credit_grants": { + "name": "billing_credit_grants", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "billing_credit_grants_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "amount_cents": { + "name": "amount_cents", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor": { + "name": "actor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'admin_script'" + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "applies_at": { + "name": "applies_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "billing_credit_grants_org_created_idx": { + "name": "billing_credit_grants_org_created_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "billing_credit_grants_org_idempotency_idx": { + "name": "billing_credit_grants_org_idempotency_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "billing_credit_grants_org_id_organizations_id_fk": { + "name": "billing_credit_grants_org_id_organizations_id_fk", + "tableFrom": "billing_credit_grants", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "billing_credit_grants_amount_cents_positive": { + "name": "billing_credit_grants_amount_cents_positive", + "value": "\"billing_credit_grants\".\"amount_cents\" > 0" + }, + "billing_credit_grants_reason_nonempty": { + "name": "billing_credit_grants_reason_nonempty", + "value": "length(btrim(\"billing_credit_grants\".\"reason\")) > 0" + }, + "billing_credit_grants_actor_nonempty": { + "name": "billing_credit_grants_actor_nonempty", + "value": "length(btrim(\"billing_credit_grants\".\"actor\")) > 0" + }, + "billing_credit_grants_source_nonempty": { + "name": "billing_credit_grants_source_nonempty", + "value": "length(btrim(\"billing_credit_grants\".\"source\")) > 0" + }, + "billing_credit_grants_idempotency_key_nonempty": { + "name": "billing_credit_grants_idempotency_key_nonempty", + "value": "length(btrim(\"billing_credit_grants\".\"idempotency_key\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.billing_provider_events": { + "name": "billing_provider_events", + "schema": "", + "columns": { + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'paddle'" + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_object_id": { + "name": "provider_object_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "billing_provider_events_org_occurred_idx": { + "name": "billing_provider_events_org_occurred_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "billing_provider_events_type_occurred_idx": { + "name": "billing_provider_events_type_occurred_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "billing_provider_events_org_id_organizations_id_fk": { + "name": "billing_provider_events_org_id_organizations_id_fk", + "tableFrom": "billing_provider_events", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "billing_provider_events_provider_check": { + "name": "billing_provider_events_provider_check", + "value": "\"billing_provider_events\".\"provider\" = 'paddle'" + }, + "billing_provider_events_outcome_check": { + "name": "billing_provider_events_outcome_check", + "value": "\"billing_provider_events\".\"outcome\" IN ('processing', 'applied', 'stale', 'ignored', 'unmatched')" + }, + "billing_provider_events_event_type_nonempty": { + "name": "billing_provider_events_event_type_nonempty", + "value": "length(btrim(\"billing_provider_events\".\"event_type\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.billing_provider_subscriptions": { + "name": "billing_provider_subscriptions", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'paddle'" + }, + "provider_subscription_id": { + "name": "provider_subscription_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_customer_id": { + "name": "provider_customer_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "current_period_starts_at": { + "name": "current_period_starts_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "current_period_ends_at": { + "name": "current_period_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "latest_event_occurred_at": { + "name": "latest_event_occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "latest_event_id": { + "name": "latest_event_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "billing_provider_subscriptions_provider_id_idx": { + "name": "billing_provider_subscriptions_provider_id_idx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_subscription_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "billing_provider_subscriptions_status_period_idx": { + "name": "billing_provider_subscriptions_status_period_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "current_period_ends_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "billing_provider_subscriptions_org_id_organizations_id_fk": { + "name": "billing_provider_subscriptions_org_id_organizations_id_fk", + "tableFrom": "billing_provider_subscriptions", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "billing_provider_subscriptions_provider_check": { + "name": "billing_provider_subscriptions_provider_check", + "value": "\"billing_provider_subscriptions\".\"provider\" = 'paddle'" + }, + "billing_provider_subscriptions_status_check": { + "name": "billing_provider_subscriptions_status_check", + "value": "\"billing_provider_subscriptions\".\"status\" IN ('active', 'trialing', 'past_due', 'paused', 'canceled')" + }, + "billing_provider_subscriptions_provider_subscription_nonempty": { + "name": "billing_provider_subscriptions_provider_subscription_nonempty", + "value": "length(btrim(\"billing_provider_subscriptions\".\"provider_subscription_id\")) > 0" + }, + "billing_provider_subscriptions_provider_customer_nonempty": { + "name": "billing_provider_subscriptions_provider_customer_nonempty", + "value": "length(btrim(\"billing_provider_subscriptions\".\"provider_customer_id\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.cli_device_authorizations": { + "name": "cli_device_authorizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "cli_device_authorizations_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "device_code_sha256": { + "name": "device_code_sha256", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "user_code": { + "name": "user_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "user_id": { + "name": "user_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "token_id": { + "name": "token_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "approved_at": { + "name": "approved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "poll_count": { + "name": "poll_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "cli_device_authorizations_device_code_sha256_idx": { + "name": "cli_device_authorizations_device_code_sha256_idx", + "columns": [ + { + "expression": "device_code_sha256", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "cli_device_authorizations_user_code_idx": { + "name": "cli_device_authorizations_user_code_idx", + "columns": [ + { + "expression": "user_code", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "cli_device_authorizations_user_id_users_id_fk": { + "name": "cli_device_authorizations_user_id_users_id_fk", + "tableFrom": "cli_device_authorizations", + "columnsFrom": [ + "user_id" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + }, + "cli_device_authorizations_org_id_organizations_id_fk": { + "name": "cli_device_authorizations_org_id_organizations_id_fk", + "tableFrom": "cli_device_authorizations", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + }, + "cli_device_authorizations_token_id_cli_tokens_id_fk": { + "name": "cli_device_authorizations_token_id_cli_tokens_id_fk", + "tableFrom": "cli_device_authorizations", + "columnsFrom": [ + "token_id" + ], + "tableTo": "cli_tokens", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "cli_device_authorizations_status_check": { + "name": "cli_device_authorizations_status_check", + "value": "\"cli_device_authorizations\".\"status\" IN ('pending', 'approved', 'denied', 'claimed')" + } + }, + "isRLSEnabled": false + }, + "public.cli_refresh_sessions": { + "name": "cli_refresh_sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "cli_refresh_sessions_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "user_id": { + "name": "user_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "cli_refresh_sessions_expiry_idx": { + "name": "cli_refresh_sessions_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "cli_refresh_sessions_user_id_users_id_fk": { + "name": "cli_refresh_sessions_user_id_users_id_fk", + "tableFrom": "cli_refresh_sessions", + "columnsFrom": [ + "user_id" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "cli_refresh_sessions_org_id_organizations_id_fk": { + "name": "cli_refresh_sessions_org_id_organizations_id_fk", + "tableFrom": "cli_refresh_sessions", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "cli_refresh_sessions_expiry_check": { + "name": "cli_refresh_sessions_expiry_check", + "value": "\"cli_refresh_sessions\".\"expires_at\" > \"cli_refresh_sessions\".\"created_at\"" + } + }, + "isRLSEnabled": false + }, + "public.cli_refresh_tokens": { + "name": "cli_refresh_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "cli_refresh_tokens_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "token_sha256": { + "name": "token_sha256", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "cli_refresh_tokens_token_sha256_idx": { + "name": "cli_refresh_tokens_token_sha256_idx", + "columns": [ + { + "expression": "token_sha256", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "cli_refresh_tokens_current_session_idx": { + "name": "cli_refresh_tokens_current_session_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "where": "\"cli_refresh_tokens\".\"consumed_at\" IS NULL", + "concurrently": false + }, + "cli_refresh_tokens_session_idx": { + "name": "cli_refresh_tokens_session_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "cli_refresh_tokens_session_id_cli_refresh_sessions_id_fk": { + "name": "cli_refresh_tokens_session_id_cli_refresh_sessions_id_fk", + "tableFrom": "cli_refresh_tokens", + "columnsFrom": [ + "session_id" + ], + "tableTo": "cli_refresh_sessions", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "cli_refresh_tokens_expiry_check": { + "name": "cli_refresh_tokens_expiry_check", + "value": "\"cli_refresh_tokens\".\"expires_at\" > \"cli_refresh_tokens\".\"created_at\"" + }, + "cli_refresh_tokens_consumed_after_created_check": { + "name": "cli_refresh_tokens_consumed_after_created_check", + "value": "\"cli_refresh_tokens\".\"consumed_at\" IS NULL OR \"cli_refresh_tokens\".\"consumed_at\" >= \"cli_refresh_tokens\".\"created_at\"" + } + }, + "isRLSEnabled": false + }, + "public.cli_tokens": { + "name": "cli_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "cli_tokens_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "token_sha256": { + "name": "token_sha256", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "refresh_session_id": { + "name": "refresh_session_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "cli_tokens_token_sha256_idx": { + "name": "cli_tokens_token_sha256_idx", + "columns": [ + { + "expression": "token_sha256", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "cli_tokens_org_created_idx": { + "name": "cli_tokens_org_created_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "cli_tokens_refresh_session_idx": { + "name": "cli_tokens_refresh_session_idx", + "columns": [ + { + "expression": "refresh_session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "cli_tokens_user_id_users_id_fk": { + "name": "cli_tokens_user_id_users_id_fk", + "tableFrom": "cli_tokens", + "columnsFrom": [ + "user_id" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "cli_tokens_org_id_organizations_id_fk": { + "name": "cli_tokens_org_id_organizations_id_fk", + "tableFrom": "cli_tokens", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "cli_tokens_refresh_session_id_cli_refresh_sessions_id_fk": { + "name": "cli_tokens_refresh_session_id_cli_refresh_sessions_id_fk", + "tableFrom": "cli_tokens", + "columnsFrom": [ + "refresh_session_id" + ], + "tableTo": "cli_refresh_sessions", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "cli_tokens_scope_check": { + "name": "cli_tokens_scope_check", + "value": "\"cli_tokens\".\"scope\" IN ('inference')" + } + }, + "isRLSEnabled": false + }, + "public.customer_notification_email_deliveries": { + "name": "customer_notification_email_deliveries", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "email_category": { + "name": "email_category", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_count": { + "name": "event_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_attempt_at": { + "name": "last_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "delivered_at": { + "name": "delivered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "customer_notification_email_deliveries_status_created_idx": { + "name": "customer_notification_email_deliveries_status_created_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "customer_notification_email_deliveries_org_created_idx": { + "name": "customer_notification_email_deliveries_org_created_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "customer_notification_email_deliveries_org_id_organizations_id_fk": { + "name": "customer_notification_email_deliveries_org_id_organizations_id_fk", + "tableFrom": "customer_notification_email_deliveries", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "customer_notification_email_deliveries_category_check": { + "name": "customer_notification_email_deliveries_category_check", + "value": "\"customer_notification_email_deliveries\".\"email_category\" IN ('security', 'payment_failure', 'trial_expiry', 'service_incident', 'billing_summary')" + }, + "customer_notification_email_deliveries_status_check": { + "name": "customer_notification_email_deliveries_status_check", + "value": "\"customer_notification_email_deliveries\".\"status\" IN ('queued', 'retrying', 'sending', 'delivered', 'suppressed', 'failed')" + }, + "customer_notification_email_deliveries_event_count_check": { + "name": "customer_notification_email_deliveries_event_count_check", + "value": "\"customer_notification_email_deliveries\".\"event_count\" BETWEEN 1 AND 20" + } + }, + "isRLSEnabled": false + }, + "public.customer_notification_email_delivery_events": { + "name": "customer_notification_email_delivery_events", + "schema": "", + "columns": { + "event_id": { + "name": "event_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "delivery_id": { + "name": "delivery_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "customer_notification_email_delivery_events_delivery_idx": { + "name": "customer_notification_email_delivery_events_delivery_idx", + "columns": [ + { + "expression": "delivery_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "customer_notification_email_delivery_events_delivery_id_customer_notification_email_deliveries_id_fk": { + "name": "customer_notification_email_delivery_events_delivery_id_customer_notification_email_deliveries_id_fk", + "tableFrom": "customer_notification_email_delivery_events", + "columnsFrom": [ + "delivery_id" + ], + "tableTo": "customer_notification_email_deliveries", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.customer_notification_events": { + "name": "customer_notification_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "customer_notification_events_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action_label": { + "name": "action_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action_href": { + "name": "action_href", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "visibility": { + "name": "visibility", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "customer_notification_events_org_key_idx": { + "name": "customer_notification_events_org_key_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "customer_notification_events_org_created_idx": { + "name": "customer_notification_events_org_created_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "customer_notification_events_expiry_idx": { + "name": "customer_notification_events_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "customer_notification_events_org_id_organizations_id_fk": { + "name": "customer_notification_events_org_id_organizations_id_fk", + "tableFrom": "customer_notification_events", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "customer_notification_events_severity_check": { + "name": "customer_notification_events_severity_check", + "value": "\"customer_notification_events\".\"severity\" IN ('info', 'warning', 'critical')" + }, + "customer_notification_events_category_check": { + "name": "customer_notification_events_category_check", + "value": "\"customer_notification_events\".\"category\" IN ('trial', 'billing', 'service', 'security')" + }, + "customer_notification_events_visibility_check": { + "name": "customer_notification_events_visibility_check", + "value": "\"customer_notification_events\".\"visibility\" IN ('members', 'admins')" + }, + "customer_notification_events_content_check": { + "name": "customer_notification_events_content_check", + "value": "length(btrim(\"customer_notification_events\".\"idempotency_key\")) BETWEEN 1 AND 200 AND length(btrim(\"customer_notification_events\".\"title\")) BETWEEN 1 AND 120 AND length(btrim(\"customer_notification_events\".\"body\")) BETWEEN 1 AND 500" + }, + "customer_notification_events_action_check": { + "name": "customer_notification_events_action_check", + "value": "(\"customer_notification_events\".\"action_label\" IS NULL AND \"customer_notification_events\".\"action_href\" IS NULL) OR (\"customer_notification_events\".\"action_label\" IS NOT NULL AND \"customer_notification_events\".\"action_href\" IS NOT NULL AND length(btrim(\"customer_notification_events\".\"action_label\")) BETWEEN 1 AND 60 AND \"customer_notification_events\".\"action_href\" ~ '^/orgs/')" + }, + "customer_notification_events_expiry_check": { + "name": "customer_notification_events_expiry_check", + "value": "\"customer_notification_events\".\"expires_at\" > \"customer_notification_events\".\"created_at\"" + } + }, + "isRLSEnabled": false + }, + "public.customer_notification_reads": { + "name": "customer_notification_reads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "customer_notification_reads_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "event_id": { + "name": "event_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "read_at": { + "name": "read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "customer_notification_reads_event_user_idx": { + "name": "customer_notification_reads_event_user_idx", + "columns": [ + { + "expression": "event_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "customer_notification_reads_user_event_idx": { + "name": "customer_notification_reads_user_event_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "customer_notification_reads_event_id_customer_notification_events_id_fk": { + "name": "customer_notification_reads_event_id_customer_notification_events_id_fk", + "tableFrom": "customer_notification_reads", + "columnsFrom": [ + "event_id" + ], + "tableTo": "customer_notification_events", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "customer_notification_reads_user_id_users_id_fk": { + "name": "customer_notification_reads_user_id_users_id_fk", + "tableFrom": "customer_notification_reads", + "columnsFrom": [ + "user_id" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.finding_approvals": { + "name": "finding_approvals", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "review_id": { + "name": "review_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "finding_id": { + "name": "finding_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "actor_github_id": { + "name": "actor_github_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_login_snapshot": { + "name": "actor_login_snapshot", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_role_snapshot": { + "name": "actor_role_snapshot", + "type": "finding_approval_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "verb": { + "name": "verb", + "type": "finding_approval_verb", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'approve'" + }, + "reason_tag": { + "name": "reason_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_self_dismissal": { + "name": "author_self_dismissal", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "finding_kind": { + "name": "finding_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "finding_severity": { + "name": "finding_severity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "finding_confidence": { + "name": "finding_confidence", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "finding_model": { + "name": "finding_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "finding_scorer_model": { + "name": "finding_scorer_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rationale": { + "name": "rationale", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "finding_approval_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "source_comment_id": { + "name": "source_comment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_org_id": { + "name": "source_org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_repository_id": { + "name": "source_repository_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_github_installation_id": { + "name": "source_github_installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_github_repo_id": { + "name": "source_github_repo_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_pr_number": { + "name": "source_pr_number", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "source_head_sha": { + "name": "source_head_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_webhook_delivery_id": { + "name": "source_webhook_delivery_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_github_comment_id": { + "name": "source_github_comment_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_comment_kind": { + "name": "source_comment_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_binding_state": { + "name": "source_binding_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'exact'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_by_user_id": { + "name": "revoked_by_user_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "finding_approvals_active_idx": { + "name": "finding_approvals_active_idx", + "columns": [ + { + "expression": "review_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "finding_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "where": "\"finding_approvals\".\"revoked_at\" IS NULL", + "concurrently": false + }, + "finding_approvals_github_comment_idx": { + "name": "finding_approvals_github_comment_idx", + "columns": [ + { + "expression": "source_github_installation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_github_repo_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_comment_kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_github_comment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "where": "\"finding_approvals\".\"source\" = 'github'", + "concurrently": false + }, + "finding_approvals_github_delivery_idx": { + "name": "finding_approvals_github_delivery_idx", + "columns": [ + { + "expression": "source_webhook_delivery_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "where": "\"finding_approvals\".\"source\" = 'github'", + "concurrently": false + }, + "finding_approvals_review_idx": { + "name": "finding_approvals_review_idx", + "columns": [ + { + "expression": "review_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "finding_approvals_review_id_reviews_id_fk": { + "name": "finding_approvals_review_id_reviews_id_fk", + "tableFrom": "finding_approvals", + "columnsFrom": [ + "review_id" + ], + "tableTo": "reviews", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "finding_approvals_actor_user_id_users_id_fk": { + "name": "finding_approvals_actor_user_id_users_id_fk", + "tableFrom": "finding_approvals", + "columnsFrom": [ + "actor_user_id" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "finding_approvals_revoked_by_user_id_users_id_fk": { + "name": "finding_approvals_revoked_by_user_id_users_id_fk", + "tableFrom": "finding_approvals", + "columnsFrom": [ + "revoked_by_user_id" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "finding_approvals_rationale_nonempty": { + "name": "finding_approvals_rationale_nonempty", + "value": "length(btrim(\"finding_approvals\".\"rationale\")) > 0" + }, + "finding_approvals_dismissal_check": { + "name": "finding_approvals_dismissal_check", + "value": "(\"finding_approvals\".\"verb\" = 'approve' AND \"finding_approvals\".\"reason_tag\" IS NULL AND \"finding_approvals\".\"author_self_dismissal\" = false AND \"finding_approvals\".\"finding_kind\" IS NULL AND \"finding_approvals\".\"finding_severity\" IS NULL AND \"finding_approvals\".\"finding_confidence\" IS NULL AND \"finding_approvals\".\"finding_model\" IS NULL AND \"finding_approvals\".\"finding_scorer_model\" IS NULL) OR (\"finding_approvals\".\"verb\" = 'dismiss' AND \"finding_approvals\".\"reason_tag\" IS NOT NULL AND \"finding_approvals\".\"reason_tag\" IN ('false-positive', 'accepted-risk', 'out-of-scope') AND \"finding_approvals\".\"finding_kind\" IS NOT NULL AND \"finding_approvals\".\"finding_severity\" IS NOT NULL AND \"finding_approvals\".\"finding_confidence\" IS NOT NULL AND \"finding_approvals\".\"finding_confidence\" BETWEEN 0 AND 1 AND \"finding_approvals\".\"finding_model\" IS NOT NULL)" + }, + "finding_approvals_binding_check": { + "name": "finding_approvals_binding_check", + "value": "(\"finding_approvals\".\"source_binding_state\" = 'legacy' AND \"finding_approvals\".\"source_org_id\" IS NULL AND \"finding_approvals\".\"source_repository_id\" IS NULL AND \"finding_approvals\".\"source_github_installation_id\" IS NULL AND \"finding_approvals\".\"source_github_repo_id\" IS NULL AND \"finding_approvals\".\"source_pr_number\" IS NULL AND \"finding_approvals\".\"source_head_sha\" IS NULL) OR (\"finding_approvals\".\"source_binding_state\" = 'exact' AND \"finding_approvals\".\"source_org_id\" > 0 AND \"finding_approvals\".\"source_repository_id\" > 0 AND \"finding_approvals\".\"source_github_installation_id\" > 0 AND \"finding_approvals\".\"source_github_repo_id\" > 0 AND \"finding_approvals\".\"source_pr_number\" > 0 AND length(btrim(\"finding_approvals\".\"source_head_sha\")) BETWEEN 1 AND 200)" + }, + "finding_approvals_github_source_check": { + "name": "finding_approvals_github_source_check", + "value": "\"finding_approvals\".\"source\" <> 'github' OR (\"finding_approvals\".\"source_webhook_delivery_id\" IS NULL AND \"finding_approvals\".\"source_github_comment_id\" IS NULL AND \"finding_approvals\".\"source_comment_kind\" IS NULL) OR (length(btrim(\"finding_approvals\".\"source_webhook_delivery_id\")) BETWEEN 1 AND 200 AND \"finding_approvals\".\"source_github_comment_id\" > 0 AND \"finding_approvals\".\"source_comment_kind\" IN ('issue_comment', 'pull_request_review_comment'))" + } + }, + "isRLSEnabled": false + }, + "public.finding_feedback": { + "name": "finding_feedback", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "finding_feedback_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "finding_publication_id": { + "name": "finding_publication_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_github_comment_id": { + "name": "source_github_comment_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_github_reaction_id": { + "name": "source_github_reaction_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "reaction_content": { + "name": "reaction_content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_github_id": { + "name": "actor_github_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "actor_login_snapshot": { + "name": "actor_login_snapshot", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pr_author_github_id": { + "name": "pr_author_github_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "pr_author_login_snapshot": { + "name": "pr_author_login_snapshot", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_is_pr_author": { + "name": "actor_is_pr_author", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "observed_at": { + "name": "observed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "source_delivery_id": { + "name": "source_delivery_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "suggested_reason_tag": { + "name": "suggested_reason_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "finding_feedback_publication_observed_idx": { + "name": "finding_feedback_publication_observed_idx", + "columns": [ + { + "expression": "finding_publication_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "observed_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "finding_feedback_github_reply_idx": { + "name": "finding_feedback_github_reply_idx", + "columns": [ + { + "expression": "source_github_comment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "where": "\"finding_feedback\".\"source\" = 'reply'", + "concurrently": false + }, + "finding_feedback_github_reaction_idx": { + "name": "finding_feedback_github_reaction_idx", + "columns": [ + { + "expression": "source_github_reaction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "where": "\"finding_feedback\".\"source\" = 'reaction'", + "concurrently": false + } + }, + "foreignKeys": { + "finding_feedback_finding_publication_id_finding_publications_id_fk": { + "name": "finding_feedback_finding_publication_id_finding_publications_id_fk", + "tableFrom": "finding_feedback", + "columnsFrom": [ + "finding_publication_id" + ], + "tableTo": "finding_publications", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "finding_feedback_source_check": { + "name": "finding_feedback_source_check", + "value": "\"finding_feedback\".\"source\" IN ('reply', 'reaction')" + }, + "finding_feedback_identity_check": { + "name": "finding_feedback_identity_check", + "value": "(\"finding_feedback\".\"source\" = 'reply' AND \"finding_feedback\".\"source_github_comment_id\" IS NOT NULL AND \"finding_feedback\".\"source_github_comment_id\" BETWEEN 1 AND 9007199254740991 AND \"finding_feedback\".\"source_github_reaction_id\" IS NULL AND \"finding_feedback\".\"reaction_content\" IS NULL AND \"finding_feedback\".\"body\" IS NOT NULL AND length(btrim(\"finding_feedback\".\"body\")) BETWEEN 1 AND 65535 AND length(btrim(\"finding_feedback\".\"source_delivery_id\")) BETWEEN 1 AND 200) OR (\"finding_feedback\".\"source\" = 'reaction' AND \"finding_feedback\".\"source_github_comment_id\" IS NOT NULL AND \"finding_feedback\".\"source_github_comment_id\" BETWEEN 1 AND 9007199254740991 AND \"finding_feedback\".\"source_github_reaction_id\" IS NOT NULL AND \"finding_feedback\".\"source_github_reaction_id\" BETWEEN 1 AND 9007199254740991 AND \"finding_feedback\".\"reaction_content\" IS NOT NULL AND \"finding_feedback\".\"reaction_content\" IN ('+1', '-1', 'unknown') AND \"finding_feedback\".\"body\" IS NULL AND \"finding_feedback\".\"source_delivery_id\" IS NULL)" + }, + "finding_feedback_actor_check": { + "name": "finding_feedback_actor_check", + "value": "\"finding_feedback\".\"actor_github_id\" BETWEEN 1 AND 9007199254740991 AND length(btrim(\"finding_feedback\".\"actor_login_snapshot\")) BETWEEN 1 AND 100 AND \"finding_feedback\".\"pr_author_github_id\" BETWEEN 1 AND 9007199254740991 AND length(btrim(\"finding_feedback\".\"pr_author_login_snapshot\")) BETWEEN 1 AND 100 AND \"finding_feedback\".\"actor_is_pr_author\" = (\"finding_feedback\".\"actor_github_id\" = \"finding_feedback\".\"pr_author_github_id\")" + }, + "finding_feedback_suggested_reason_check": { + "name": "finding_feedback_suggested_reason_check", + "value": "\"finding_feedback\".\"suggested_reason_tag\" IS NULL OR \"finding_feedback\".\"suggested_reason_tag\" IN ('false-positive', 'accepted-risk', 'out-of-scope')" + } + }, + "isRLSEnabled": false + }, + "public.finding_feedback_reconciliations": { + "name": "finding_feedback_reconciliations", + "schema": "", + "columns": { + "finding_publication_id": { + "name": "finding_publication_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_reconcile_at": { + "name": "next_reconcile_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_attempt_at": { + "name": "last_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_successful_at": { + "name": "last_successful_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "finding_feedback_reconcile_due_idx": { + "name": "finding_feedback_reconcile_due_idx", + "columns": [ + { + "expression": "next_reconcile_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "finding_feedback_reconciliations_finding_publication_id_finding_publications_id_fk": { + "name": "finding_feedback_reconciliations_finding_publication_id_finding_publications_id_fk", + "tableFrom": "finding_feedback_reconciliations", + "columnsFrom": [ + "finding_publication_id" + ], + "tableTo": "finding_publications", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "finding_feedback_reconcile_attempt_count_check": { + "name": "finding_feedback_reconcile_attempt_count_check", + "value": "\"finding_feedback_reconciliations\".\"attempt_count\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.finding_publications": { + "name": "finding_publications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "finding_publications_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "review_id": { + "name": "review_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "finding_id": { + "name": "finding_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "stable_identity": { + "name": "stable_identity", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "initial_state": { + "name": "initial_state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "current_state": { + "name": "current_state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "github_comment_id": { + "name": "github_comment_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lifecycle_observed_at": { + "name": "lifecycle_observed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "finding_publications_review_finding_idx": { + "name": "finding_publications_review_finding_idx", + "columns": [ + { + "expression": "review_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "finding_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "finding_publications_comment_idx": { + "name": "finding_publications_comment_idx", + "columns": [ + { + "expression": "github_comment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "finding_publications_stable_finding_idx": { + "name": "finding_publications_stable_finding_idx", + "columns": [ + { + "expression": "finding_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "stable_identity", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "finding_publications_review_id_reviews_id_fk": { + "name": "finding_publications_review_id_reviews_id_fk", + "tableFrom": "finding_publications", + "columnsFrom": [ + "review_id" + ], + "tableTo": "reviews", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "finding_publications_finding_id_check": { + "name": "finding_publications_finding_id_check", + "value": "length(btrim(\"finding_publications\".\"finding_id\")) BETWEEN 1 AND 500" + }, + "finding_publications_initial_state_check": { + "name": "finding_publications_initial_state_check", + "value": "\"finding_publications\".\"initial_state\" IN ('inline', 'fileComment', 'checkAnnotation', 'summaryOnly', 'carried', 'resolved', 'suppressed', 'inlineRejected', 'unknown')" + }, + "finding_publications_current_state_check": { + "name": "finding_publications_current_state_check", + "value": "\"finding_publications\".\"current_state\" IN ('inline', 'fileComment', 'checkAnnotation', 'summaryOnly', 'carried', 'resolved', 'suppressed', 'inlineRejected', 'outdated', 'deleted', 'unknown')" + }, + "finding_publications_github_comment_id_check": { + "name": "finding_publications_github_comment_id_check", + "value": "\"finding_publications\".\"github_comment_id\" IS NULL OR \"finding_publications\".\"github_comment_id\" ~ '^[1-9][0-9]{0,19}$'" + }, + "finding_publications_file_comment_identity_check": { + "name": "finding_publications_file_comment_identity_check", + "value": "(\"finding_publications\".\"initial_state\" <> 'fileComment' AND \"finding_publications\".\"current_state\" <> 'fileComment') OR \"finding_publications\".\"github_comment_id\" IS NOT NULL" + } + }, + "isRLSEnabled": false + }, + "public.github_webhook_delivery_recoveries": { + "name": "github_webhook_delivery_recoveries", + "schema": "", + "columns": { + "delivery_id": { + "name": "delivery_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "delivery_guid": { + "name": "delivery_guid", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delivered_at": { + "name": "delivered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redelivery": { + "name": "redelivery", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "observed_at": { + "name": "observed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "request_state": { + "name": "request_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "request_attempts": { + "name": "request_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_requested_at": { + "name": "last_requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "request_status_code": { + "name": "request_status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "recovery_delivery_id": { + "name": "recovery_delivery_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_error_category": { + "name": "last_error_category", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "github_webhook_delivery_recoveries_guid_idx": { + "name": "github_webhook_delivery_recoveries_guid_idx", + "columns": [ + { + "expression": "delivery_guid", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "delivered_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "github_webhook_delivery_recoveries_retry_idx": { + "name": "github_webhook_delivery_recoveries_retry_idx", + "columns": [ + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "delivered_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "where": "\"github_webhook_delivery_recoveries\".\"outcome\" = 'failure' AND \"github_webhook_delivery_recoveries\".\"recovery_delivery_id\" IS NULL", + "concurrently": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "github_webhook_delivery_recoveries_outcome_check": { + "name": "github_webhook_delivery_recoveries_outcome_check", + "value": "\"github_webhook_delivery_recoveries\".\"outcome\" IN ('success', 'failure', 'pending')" + }, + "github_webhook_delivery_recoveries_request_state_check": { + "name": "github_webhook_delivery_recoveries_request_state_check", + "value": "\"github_webhook_delivery_recoveries\".\"request_state\" IS NULL OR \"github_webhook_delivery_recoveries\".\"request_state\" IN ('requesting', 'retryable', 'accepted', 'terminal', 'exhausted', 'recovered')" + }, + "github_webhook_delivery_recoveries_attempts_check": { + "name": "github_webhook_delivery_recoveries_attempts_check", + "value": "\"github_webhook_delivery_recoveries\".\"request_attempts\" >= 0 AND \"github_webhook_delivery_recoveries\".\"request_attempts\" <= 2" + } + }, + "isRLSEnabled": false + }, + "public.github_webhook_redelivery_state": { + "name": "github_webhook_redelivery_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "sweep_started_at": { + "name": "sweep_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_page_at": { + "name": "last_page_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sweep_completed_at": { + "name": "last_sweep_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "rate_limited_until": { + "name": "rate_limited_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error_category": { + "name": "last_error_category", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "github_webhook_redelivery_state_singleton_check": { + "name": "github_webhook_redelivery_state_singleton_check", + "value": "\"github_webhook_redelivery_state\".\"id\" = 1" + } + }, + "isRLSEnabled": false + }, + "public.hosted_provider_keys": { + "name": "hosted_provider_keys", + "schema": "", + "columns": { + "create_intent_id": { + "name": "create_intent_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_key_name": { + "name": "provider_key_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_key_hash": { + "name": "provider_key_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "conflicting_provider_key_hash": { + "name": "conflicting_provider_key_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sealed_runtime_key": { + "name": "sealed_runtime_key", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "entitlement_period_starts_at": { + "name": "entitlement_period_starts_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "entitlement_period_ends_at": { + "name": "entitlement_period_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "entitlement_updated_at": { + "name": "entitlement_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "limit_micros": { + "name": "limit_micros", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "create_attempted_at": { + "name": "create_attempted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "create_outcome": { + "name": "create_outcome", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revocation_requested_at": { + "name": "revocation_requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoke_attempted_at": { + "name": "revoke_attempted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoke_outcome": { + "name": "revoke_outcome", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "reconciliation_required_at": { + "name": "reconciliation_required_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "lease_id": { + "name": "lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "lease_kind": { + "name": "lease_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "clock_timestamp()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "clock_timestamp()" + } + }, + "indexes": { + "hosted_provider_keys_provider_key_hash_unique": { + "name": "hosted_provider_keys_provider_key_hash_unique", + "columns": [ + { + "expression": "provider_key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "where": "\"hosted_provider_keys\".\"provider_key_hash\" IS NOT NULL", + "concurrently": false + }, + "hosted_provider_keys_entitlement_binding_unique": { + "name": "hosted_provider_keys_entitlement_binding_unique", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entitlement_period_starts_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entitlement_period_ends_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "limit_micros", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "where": "\"hosted_provider_keys\".\"state\" NOT IN ('revoked', 'cancelled')", + "concurrently": false + }, + "hosted_provider_keys_active_org_unique": { + "name": "hosted_provider_keys_active_org_unique", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "where": "\"hosted_provider_keys\".\"state\" = 'active'", + "concurrently": false + }, + "hosted_provider_keys_runtime_org_unique": { + "name": "hosted_provider_keys_runtime_org_unique", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "where": "\"hosted_provider_keys\".\"sealed_runtime_key\" IS NOT NULL", + "concurrently": false + }, + "hosted_provider_keys_reconciliation_idx": { + "name": "hosted_provider_keys_reconciliation_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "reconciliation_required_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "hosted_provider_keys_org_id_organizations_id_fk": { + "name": "hosted_provider_keys_org_id_organizations_id_fk", + "tableFrom": "hosted_provider_keys", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "restrict" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "hosted_provider_keys_provider_key_name_unique": { + "name": "hosted_provider_keys_provider_key_name_unique", + "columns": [ + "provider_key_name" + ], + "nullsNotDistinct": false + } + }, + "policies": {}, + "checkConstraints": { + "hosted_provider_keys_state_check": { + "name": "hosted_provider_keys_state_check", + "value": "\"hosted_provider_keys\".\"state\" IN ('provisioning', 'activating', 'active', 'rejected', 'orphaned', 'revocation_pending', 'revoked', 'cancelled')" + }, + "hosted_provider_keys_provider_key_name_nonempty": { + "name": "hosted_provider_keys_provider_key_name_nonempty", + "value": "length(btrim(\"hosted_provider_keys\".\"provider_key_name\")) > 0" + }, + "hosted_provider_keys_provider_key_hash_nonempty": { + "name": "hosted_provider_keys_provider_key_hash_nonempty", + "value": "\"hosted_provider_keys\".\"provider_key_hash\" IS NULL OR length(btrim(\"hosted_provider_keys\".\"provider_key_hash\")) > 0" + }, + "hosted_provider_keys_conflicting_hash_nonempty": { + "name": "hosted_provider_keys_conflicting_hash_nonempty", + "value": "\"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL OR length(btrim(\"hosted_provider_keys\".\"conflicting_provider_key_hash\")) > 0" + }, + "hosted_provider_keys_entitlement_period_check": { + "name": "hosted_provider_keys_entitlement_period_check", + "value": "\"hosted_provider_keys\".\"entitlement_period_ends_at\" > \"hosted_provider_keys\".\"entitlement_period_starts_at\"" + }, + "hosted_provider_keys_limit_exact_range": { + "name": "hosted_provider_keys_limit_exact_range", + "value": "\"hosted_provider_keys\".\"limit_micros\" > 0 AND \"hosted_provider_keys\".\"limit_micros\" <= 2251799813685247" + }, + "hosted_provider_keys_create_outcome_check": { + "name": "hosted_provider_keys_create_outcome_check", + "value": "\"hosted_provider_keys\".\"create_outcome\" IS NULL OR \"hosted_provider_keys\".\"create_outcome\" IN ('created', 'rejected', 'rate_limited', 'ambiguous', 'name_present', 'name_not_unique', 'credential_persistence_failed', 'intent_changed', 'ownership_conflict')" + }, + "hosted_provider_keys_revoke_outcome_check": { + "name": "hosted_provider_keys_revoke_outcome_check", + "value": "\"hosted_provider_keys\".\"revoke_outcome\" IS NULL OR \"hosted_provider_keys\".\"revoke_outcome\" IN ('ambiguous', 'rejected', 'disabled', 'absent')" + }, + "hosted_provider_keys_lease_shape": { + "name": "hosted_provider_keys_lease_shape", + "value": "(\n \"hosted_provider_keys\".\"lease_id\" IS NULL\n AND \"hosted_provider_keys\".\"lease_kind\" IS NULL\n AND \"hosted_provider_keys\".\"lease_expires_at\" IS NULL\n ) OR (\n \"hosted_provider_keys\".\"lease_id\" IS NOT NULL\n AND \"hosted_provider_keys\".\"lease_kind\" IN ('create', 'revoke')\n AND \"hosted_provider_keys\".\"lease_expires_at\" IS NOT NULL\n )" + }, + "hosted_provider_keys_lease_state": { + "name": "hosted_provider_keys_lease_state", + "value": "\"hosted_provider_keys\".\"lease_id\" IS NULL OR (\n (\"hosted_provider_keys\".\"lease_kind\" = 'create' AND \"hosted_provider_keys\".\"state\" IN ('provisioning', 'activating', 'orphaned'))\n OR (\"hosted_provider_keys\".\"lease_kind\" = 'revoke' AND \"hosted_provider_keys\".\"state\" = 'revocation_pending')\n )" + }, + "hosted_provider_keys_lifecycle_shape": { + "name": "hosted_provider_keys_lifecycle_shape", + "value": "(\n \"hosted_provider_keys\".\"state\" = 'provisioning'\n AND \"hosted_provider_keys\".\"sealed_runtime_key\" IS NULL\n AND \"hosted_provider_keys\".\"provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"create_outcome\" IS NULL\n AND \"hosted_provider_keys\".\"revocation_requested_at\" IS NULL\n AND \"hosted_provider_keys\".\"revoke_outcome\" IS NULL\n AND \"hosted_provider_keys\".\"revoked_at\" IS NULL\n ) OR (\n \"hosted_provider_keys\".\"state\" = 'activating'\n AND \"hosted_provider_keys\".\"sealed_runtime_key\" IS NULL\n AND \"hosted_provider_keys\".\"provider_key_hash\" IS NOT NULL\n AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"create_attempted_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"create_outcome\" = 'created'\n AND \"hosted_provider_keys\".\"reconciliation_required_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"revocation_requested_at\" IS NULL\n AND \"hosted_provider_keys\".\"revoke_outcome\" IS NULL\n AND \"hosted_provider_keys\".\"revoked_at\" IS NULL\n ) OR (\n \"hosted_provider_keys\".\"state\" = 'active'\n AND \"hosted_provider_keys\".\"sealed_runtime_key\" IS NOT NULL\n AND \"hosted_provider_keys\".\"provider_key_hash\" IS NOT NULL\n AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"create_attempted_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"create_outcome\" = 'created'\n AND \"hosted_provider_keys\".\"reconciliation_required_at\" IS NULL\n AND \"hosted_provider_keys\".\"revocation_requested_at\" IS NULL\n AND \"hosted_provider_keys\".\"revoke_outcome\" IS NULL\n AND \"hosted_provider_keys\".\"revoked_at\" IS NULL\n ) OR (\n \"hosted_provider_keys\".\"state\" = 'rejected'\n AND \"hosted_provider_keys\".\"sealed_runtime_key\" IS NULL\n AND \"hosted_provider_keys\".\"provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"create_attempted_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"create_outcome\" = 'rejected'\n AND \"hosted_provider_keys\".\"reconciliation_required_at\" IS NULL\n AND \"hosted_provider_keys\".\"revocation_requested_at\" IS NULL\n AND \"hosted_provider_keys\".\"revoke_outcome\" IS NULL\n AND \"hosted_provider_keys\".\"revoked_at\" IS NULL\n ) OR (\n \"hosted_provider_keys\".\"state\" = 'orphaned'\n AND \"hosted_provider_keys\".\"sealed_runtime_key\" IS NULL\n AND \"hosted_provider_keys\".\"create_outcome\" IN ('ambiguous', 'name_present', 'name_not_unique', 'credential_persistence_failed', 'intent_changed', 'ownership_conflict')\n AND \"hosted_provider_keys\".\"reconciliation_required_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"revocation_requested_at\" IS NULL\n AND \"hosted_provider_keys\".\"revoke_outcome\" IS NULL\n AND \"hosted_provider_keys\".\"revoked_at\" IS NULL\n AND (\n (\"hosted_provider_keys\".\"create_outcome\" = 'ownership_conflict' AND \"hosted_provider_keys\".\"provider_key_hash\" IS NULL AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NOT NULL)\n OR (\"hosted_provider_keys\".\"create_outcome\" <> 'ownership_conflict' AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL)\n )\n ) OR (\n \"hosted_provider_keys\".\"state\" = 'revocation_pending'\n AND \"hosted_provider_keys\".\"sealed_runtime_key\" IS NULL\n AND \"hosted_provider_keys\".\"provider_key_hash\" IS NOT NULL\n AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"create_attempted_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"create_outcome\" IN ('created', 'ambiguous', 'credential_persistence_failed')\n AND \"hosted_provider_keys\".\"revocation_requested_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"reconciliation_required_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"revoked_at\" IS NULL\n ) OR (\n \"hosted_provider_keys\".\"state\" = 'revoked'\n AND \"hosted_provider_keys\".\"sealed_runtime_key\" IS NULL\n AND \"hosted_provider_keys\".\"provider_key_hash\" IS NOT NULL\n AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"create_attempted_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"create_outcome\" IN ('created', 'ambiguous', 'credential_persistence_failed')\n AND \"hosted_provider_keys\".\"revocation_requested_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"revoke_outcome\" IN ('disabled', 'absent')\n AND \"hosted_provider_keys\".\"revoked_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"reconciliation_required_at\" IS NULL\n AND \"hosted_provider_keys\".\"lease_id\" IS NULL\n ) OR (\n \"hosted_provider_keys\".\"state\" = 'cancelled'\n AND \"hosted_provider_keys\".\"sealed_runtime_key\" IS NULL\n AND \"hosted_provider_keys\".\"provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL\n AND (\n (\"hosted_provider_keys\".\"create_attempted_at\" IS NULL AND \"hosted_provider_keys\".\"create_outcome\" IS NULL)\n OR (\"hosted_provider_keys\".\"create_attempted_at\" IS NOT NULL AND \"hosted_provider_keys\".\"create_outcome\" = 'rate_limited')\n )\n AND \"hosted_provider_keys\".\"revocation_requested_at\" IS NULL\n AND \"hosted_provider_keys\".\"revoke_outcome\" IS NULL\n AND \"hosted_provider_keys\".\"revoked_at\" IS NULL\n AND \"hosted_provider_keys\".\"reconciliation_required_at\" IS NULL\n AND \"hosted_provider_keys\".\"lease_id\" IS NULL\n )" + } + }, + "isRLSEnabled": false + }, + "public.hosted_usage_reservations": { + "name": "hosted_usage_reservations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "review_id": { + "name": "review_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'review'" + }, + "reserved_micros": { + "name": "reserved_micros", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "actual_micros": { + "name": "actual_micros", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "hosted_usage_reservations_review_idx": { + "name": "hosted_usage_reservations_review_idx", + "columns": [ + { + "expression": "review_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "hosted_usage_reservations_active_org_expiry_idx": { + "name": "hosted_usage_reservations_active_org_expiry_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "where": "\"hosted_usage_reservations\".\"status\" = 'active'", + "concurrently": false + } + }, + "foreignKeys": { + "hosted_usage_reservations_org_id_organizations_id_fk": { + "name": "hosted_usage_reservations_org_id_organizations_id_fk", + "tableFrom": "hosted_usage_reservations", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "hosted_usage_reservations_review_id_reviews_id_fk": { + "name": "hosted_usage_reservations_review_id_reviews_id_fk", + "tableFrom": "hosted_usage_reservations", + "columnsFrom": [ + "review_id" + ], + "tableTo": "reviews", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "hosted_usage_reservations_status_check": { + "name": "hosted_usage_reservations_status_check", + "value": "\"hosted_usage_reservations\".\"status\" IN ('active', 'reconciled', 'released')" + }, + "hosted_usage_reservations_operation_check": { + "name": "hosted_usage_reservations_operation_check", + "value": "\"hosted_usage_reservations\".\"operation\" IN ('review', 'respond', 'cli_gateway')" + }, + "hosted_usage_reservations_operation_reference_check": { + "name": "hosted_usage_reservations_operation_reference_check", + "value": "(\"hosted_usage_reservations\".\"operation\" = 'review' AND \"hosted_usage_reservations\".\"review_id\" IS NOT NULL) OR (\"hosted_usage_reservations\".\"operation\" IN ('respond', 'cli_gateway') AND \"hosted_usage_reservations\".\"review_id\" IS NULL)" + }, + "hosted_usage_reservations_reserved_positive": { + "name": "hosted_usage_reservations_reserved_positive", + "value": "\"hosted_usage_reservations\".\"reserved_micros\" > 0" + }, + "hosted_usage_reservations_actual_nonnegative": { + "name": "hosted_usage_reservations_actual_nonnegative", + "value": "\"hosted_usage_reservations\".\"actual_micros\" IS NULL OR \"hosted_usage_reservations\".\"actual_micros\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.ilert_alert_events": { + "name": "ilert_alert_events", + "schema": "", + "columns": { + "sequence": { + "name": "sequence", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "ilert_alert_events_sequence_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "event_id": { + "name": "event_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "alert_id": { + "name": "alert_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "priority": { + "name": "priority", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "alert_source_id": { + "name": "alert_source_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "alert_source_name": { + "name": "alert_source_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "report_time": { + "name": "report_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "payload_sha256": { + "name": "payload_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "ilert_alert_events_event_id_idx": { + "name": "ilert_alert_events_event_id_idx", + "columns": [ + { + "expression": "event_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "ilert_alert_events_alert_sequence_idx": { + "name": "ilert_alert_events_alert_sequence_idx", + "columns": [ + { + "expression": "alert_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sequence", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "ilert_alert_events_alert_id_check": { + "name": "ilert_alert_events_alert_id_check", + "value": "\"ilert_alert_events\".\"alert_id\" ~ '^[1-9][0-9]{0,63}$'" + }, + "ilert_alert_events_event_type_check": { + "name": "ilert_alert_events_event_type_check", + "value": "length(\"ilert_alert_events\".\"event_type\") <= 64 AND \"ilert_alert_events\".\"event_type\" ~ '^alert-[a-z]+(-[a-z]+)*$'" + }, + "ilert_alert_events_status_check": { + "name": "ilert_alert_events_status_check", + "value": "\"ilert_alert_events\".\"status\" IN ('PENDING', 'ACCEPTED', 'RESOLVED')" + }, + "ilert_alert_events_priority_check": { + "name": "ilert_alert_events_priority_check", + "value": "\"ilert_alert_events\".\"priority\" IN ('HIGH', 'LOW')" + }, + "ilert_alert_events_summary_check": { + "name": "ilert_alert_events_summary_check", + "value": "length(\"ilert_alert_events\".\"summary\") BETWEEN 1 AND 512" + }, + "ilert_alert_events_details_check": { + "name": "ilert_alert_events_details_check", + "value": "length(\"ilert_alert_events\".\"details\") BETWEEN 0 AND 8192" + }, + "ilert_alert_events_source_name_check": { + "name": "ilert_alert_events_source_name_check", + "value": "length(\"ilert_alert_events\".\"alert_source_name\") BETWEEN 1 AND 256" + }, + "ilert_alert_events_payload_sha256_check": { + "name": "ilert_alert_events_payload_sha256_check", + "value": "\"ilert_alert_events\".\"payload_sha256\" ~ '^[0-9a-f]{64}$'" + } + }, + "isRLSEnabled": false + }, + "public.installations": { + "name": "installations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "installations_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "github_installation_id": { + "name": "github_installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "account_login": { + "name": "account_login", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suspended": { + "name": "suspended", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "installations_org_idx": { + "name": "installations_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "installations_org_id_organizations_id_fk": { + "name": "installations_org_id_organizations_id_fk", + "tableFrom": "installations", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "installations_github_installation_id_unique": { + "name": "installations_github_installation_id_unique", + "columns": [ + "github_installation_id" + ], + "nullsNotDistinct": false + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.jobs": { + "name": "jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "jobs_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "job_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 3 + }, + "run_after": { + "name": "run_after", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "locked_at": { + "name": "locked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "locked_by": { + "name": "locked_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "jobs_claim_idx": { + "name": "jobs_claim_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_after", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "jobs_running_locked_at_idx": { + "name": "jobs_running_locked_at_idx", + "columns": [ + { + "expression": "locked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "where": "\"jobs\".\"status\" = 'running'", + "concurrently": false + }, + "jobs_running_org_concurrency_idx": { + "name": "jobs_running_org_concurrency_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "(\"payload\"->>'sourceOrgId')", + "isExpression": true, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "where": "\"jobs\".\"status\" = 'running'", + "concurrently": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.large_review_attempts": { + "name": "large_review_attempts", + "schema": "", + "columns": { + "attempt_key": { + "name": "attempt_key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "run_key": { + "name": "run_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_sha256": { + "name": "request_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "batch_identity": { + "name": "batch_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "lease_id": { + "name": "lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "response_status": { + "name": "response_status", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "response_headers": { + "name": "response_headers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "response_body": { + "name": "response_body", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "large_review_attempts_run_request_attempt_idx": { + "name": "large_review_attempts_run_request_attempt_idx", + "columns": [ + { + "expression": "run_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_sha256", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "attempt", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "large_review_attempts_pending_request_idx": { + "name": "large_review_attempts_pending_request_idx", + "columns": [ + { + "expression": "run_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_sha256", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "where": "\"large_review_attempts\".\"state\" = 'pending'", + "concurrently": false + }, + "large_review_attempts_run_idx": { + "name": "large_review_attempts_run_idx", + "columns": [ + { + "expression": "run_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "large_review_attempts_run_key_large_review_runs_run_key_fk": { + "name": "large_review_attempts_run_key_large_review_runs_run_key_fk", + "tableFrom": "large_review_attempts", + "columnsFrom": [ + "run_key" + ], + "tableTo": "large_review_runs", + "columnsTo": [ + "run_key" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "large_review_attempts_key_check": { + "name": "large_review_attempts_key_check", + "value": "\"large_review_attempts\".\"attempt_key\" ~ '^[0-9a-f]{64}$'" + }, + "large_review_attempts_request_check": { + "name": "large_review_attempts_request_check", + "value": "\"large_review_attempts\".\"request_sha256\" ~ '^[0-9a-f]{64}$' AND \"large_review_attempts\".\"batch_identity\" ~ '^[0-9a-f]{64}$'" + }, + "large_review_attempts_attempt_check": { + "name": "large_review_attempts_attempt_check", + "value": "\"large_review_attempts\".\"attempt\" BETWEEN 1 AND 10" + }, + "large_review_attempts_state_check": { + "name": "large_review_attempts_state_check", + "value": "\"large_review_attempts\".\"state\" IN ('pending', 'completed')" + }, + "large_review_attempts_response_check": { + "name": "large_review_attempts_response_check", + "value": "(\"large_review_attempts\".\"state\" = 'pending' AND \"large_review_attempts\".\"response_status\" IS NULL AND \"large_review_attempts\".\"response_headers\" IS NULL AND \"large_review_attempts\".\"response_body\" IS NULL AND \"large_review_attempts\".\"completed_at\" IS NULL) OR (\"large_review_attempts\".\"state\" = 'completed' AND \"large_review_attempts\".\"response_status\" BETWEEN 200 AND 299 AND \"large_review_attempts\".\"response_headers\" IS NOT NULL AND \"large_review_attempts\".\"response_body\" IS NOT NULL AND \"large_review_attempts\".\"completed_at\" IS NOT NULL)" + }, + "large_review_attempts_model_check": { + "name": "large_review_attempts_model_check", + "value": "length(btrim(\"large_review_attempts\".\"model\")) BETWEEN 1 AND 500" + } + }, + "isRLSEnabled": false + }, + "public.large_review_runs": { + "name": "large_review_runs", + "schema": "", + "columns": { + "run_key": { + "name": "run_key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "current_review_id": { + "name": "current_review_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "pr_number": { + "name": "pr_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "cli_version": { + "name": "cli_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "configuration_sha256": { + "name": "configuration_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_identity": { + "name": "provider_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "head_sha": { + "name": "head_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_sha": { + "name": "base_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "retry_lineage": { + "name": "retry_lineage", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "plan_sha256": { + "name": "plan_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hosted_reservation_id": { + "name": "hosted_reservation_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "billing_state": { + "name": "billing_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "conservatively_settled_at": { + "name": "conservatively_settled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "large_review_runs_expiry_idx": { + "name": "large_review_runs_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "large_review_runs_resume_identity_idx": { + "name": "large_review_runs_resume_identity_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "pr_number", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "head_sha", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "base_sha", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cli_version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "configuration_sha256", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "retry_lineage", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "large_review_runs_current_review_id_reviews_id_fk": { + "name": "large_review_runs_current_review_id_reviews_id_fk", + "tableFrom": "large_review_runs", + "columnsFrom": [ + "current_review_id" + ], + "tableTo": "reviews", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "large_review_runs_repository_id_repositories_id_fk": { + "name": "large_review_runs_repository_id_repositories_id_fk", + "tableFrom": "large_review_runs", + "columnsFrom": [ + "repository_id" + ], + "tableTo": "repositories", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "large_review_runs_key_check": { + "name": "large_review_runs_key_check", + "value": "\"large_review_runs\".\"run_key\" ~ '^[0-9a-f]{64}$'" + }, + "large_review_runs_configuration_check": { + "name": "large_review_runs_configuration_check", + "value": "\"large_review_runs\".\"configuration_sha256\" ~ '^[0-9a-f]{64}$'" + }, + "large_review_runs_plan_check": { + "name": "large_review_runs_plan_check", + "value": "\"large_review_runs\".\"plan_sha256\" ~ '^[0-9a-f]{64}$'" + }, + "large_review_runs_identity_lengths_check": { + "name": "large_review_runs_identity_lengths_check", + "value": "\"large_review_runs\".\"pr_number\" > 0 AND length(btrim(\"large_review_runs\".\"cli_version\")) BETWEEN 1 AND 100 AND length(btrim(\"large_review_runs\".\"provider_identity\")) BETWEEN 1 AND 2048 AND length(btrim(\"large_review_runs\".\"head_sha\")) BETWEEN 1 AND 200 AND length(btrim(\"large_review_runs\".\"base_sha\")) BETWEEN 1 AND 200 AND length(btrim(\"large_review_runs\".\"retry_lineage\")) BETWEEN 1 AND 200" + }, + "large_review_runs_billing_state_check": { + "name": "large_review_runs_billing_state_check", + "value": "(\"large_review_runs\".\"billing_state\" = 'active' AND \"large_review_runs\".\"conservatively_settled_at\" IS NULL) OR (\"large_review_runs\".\"billing_state\" = 'conservative' AND \"large_review_runs\".\"conservatively_settled_at\" IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, + "public.operator_alert_deliveries": { + "name": "operator_alert_deliveries", + "schema": "", + "columns": { + "event_key": { + "name": "event_key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "github_installation_id": { + "name": "github_installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_attempt_at": { + "name": "last_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "delivered_at": { + "name": "delivered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "operator_alert_deliveries_status_created_idx": { + "name": "operator_alert_deliveries_status_created_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "operator_alert_deliveries_org_created_idx": { + "name": "operator_alert_deliveries_org_created_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "operator_alert_deliveries_org_id_organizations_id_fk": { + "name": "operator_alert_deliveries_org_id_organizations_id_fk", + "tableFrom": "operator_alert_deliveries", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "operator_alert_deliveries_event_check": { + "name": "operator_alert_deliveries_event_check", + "value": "\"operator_alert_deliveries\".\"event\" IN ('trial_started', 'trial_expired', 'installation_removed', 'subscription_started', 'subscription_past_due', 'subscription_paused', 'subscription_canceled', 'billing_anomaly', 'finding_feedback_digest')" + }, + "operator_alert_deliveries_status_check": { + "name": "operator_alert_deliveries_status_check", + "value": "\"operator_alert_deliveries\".\"status\" IN ('queued', 'retrying', 'delivered', 'failed')" + }, + "operator_alert_deliveries_event_key_nonempty": { + "name": "operator_alert_deliveries_event_key_nonempty", + "value": "length(btrim(\"operator_alert_deliveries\".\"event_key\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.org_config_probe_refreshes": { + "name": "org_config_probe_refreshes", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "refreshed_at": { + "name": "refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "org_config_probe_refreshes_org_id_organizations_id_fk": { + "name": "org_config_probe_refreshes_org_id_organizations_id_fk", + "tableFrom": "org_config_probe_refreshes", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_config_snapshots": { + "name": "org_config_snapshots", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "source_repository_id": { + "name": "source_repository_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_github_repo_id": { + "name": "source_github_repo_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "source_full_name": { + "name": "source_full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "visibility": { + "name": "visibility", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "commit_sha": { + "name": "commit_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config_yaml": { + "name": "config_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "guardrails_md": { + "name": "guardrails_md", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_policy_md": { + "name": "content_policy_md", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "files": { + "name": "files", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "loaded_files": { + "name": "loaded_files", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'::text[]" + }, + "stale": { + "name": "stale", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "org_config_snapshots_org_id_organizations_id_fk": { + "name": "org_config_snapshots_org_id_organizations_id_fk", + "tableFrom": "org_config_snapshots", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "org_config_snapshots_source_repository_id_repositories_id_fk": { + "name": "org_config_snapshots_source_repository_id_repositories_id_fk", + "tableFrom": "org_config_snapshots", + "columnsFrom": [ + "source_repository_id" + ], + "tableTo": "repositories", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_members": { + "name": "org_members", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "org_members_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'member'" + } + }, + "indexes": { + "org_members_org_user_idx": { + "name": "org_members_org_user_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "org_members_user_idx": { + "name": "org_members_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "org_members_org_id_organizations_id_fk": { + "name": "org_members_org_id_organizations_id_fk", + "tableFrom": "org_members", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "org_members_user_id_users_id_fk": { + "name": "org_members_user_id_users_id_fk", + "tableFrom": "org_members", + "columnsFrom": [ + "user_id" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_settings": { + "name": "org_settings", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "api_base": { + "name": "api_base", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_ciphertext": { + "name": "api_key_ciphertext", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "api_format": { + "name": "api_format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'openai-compatible'" + }, + "api_auth_header_ciphertext": { + "name": "api_auth_header_ciphertext", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "api_auth_value_ciphertext": { + "name": "api_auth_value_ciphertext", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model_cascade": { + "name": "model_cascade", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config_yaml": { + "name": "config_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "guardrails_md": { + "name": "guardrails_md", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_policy_md": { + "name": "content_policy_md", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "shared_config_enabled": { + "name": "shared_config_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "gate_enabled": { + "name": "gate_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "escalation_email": { + "name": "escalation_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "escalation_email_pending": { + "name": "escalation_email_pending", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "escalation_email_verified_at": { + "name": "escalation_email_verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "escalation_email_verification_token_digest": { + "name": "escalation_email_verification_token_digest", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "escalation_email_verification_token_ciphertext": { + "name": "escalation_email_verification_token_ciphertext", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "escalation_email_verification_expires_at": { + "name": "escalation_email_verification_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "escalation_email_verification_requested_at": { + "name": "escalation_email_verification_requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "escalation_email_verification_sent_at": { + "name": "escalation_email_verification_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "escalation_email_verification_message_id": { + "name": "escalation_email_verification_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "org_settings_org_id_organizations_id_fk": { + "name": "org_settings_org_id_organizations_id_fk", + "tableFrom": "org_settings", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_entitlements": { + "name": "organization_entitlements", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "subscription_mode": { + "name": "subscription_mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "trial_ends_at": { + "name": "trial_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "past_due_grace_ends_at": { + "name": "past_due_grace_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "period_starts_at": { + "name": "period_starts_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "period_ends_at": { + "name": "period_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "included_usage_micros": { + "name": "included_usage_micros", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "overage_hard_cap_micros": { + "name": "overage_hard_cap_micros", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "0" + }, + "included_usage_cents": { + "name": "included_usage_cents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "overage_hard_cap_cents": { + "name": "overage_hard_cap_cents", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "billing_contact_email": { + "name": "billing_contact_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "billing_contact_verified_at": { + "name": "billing_contact_verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "billing_contact_pending": { + "name": "billing_contact_pending", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "billing_contact_verification_token_digest": { + "name": "billing_contact_verification_token_digest", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "billing_contact_verification_token_ciphertext": { + "name": "billing_contact_verification_token_ciphertext", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "billing_contact_verification_expires_at": { + "name": "billing_contact_verification_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "billing_contact_verification_requested_at": { + "name": "billing_contact_verification_requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "billing_contact_verification_sent_at": { + "name": "billing_contact_verification_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "billing_contact_verification_message_id": { + "name": "billing_contact_verification_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "promotional_eligible": { + "name": "promotional_eligible", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "promotional_ends_at": { + "name": "promotional_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "organization_entitlements_org_id_organizations_id_fk": { + "name": "organization_entitlements_org_id_organizations_id_fk", + "tableFrom": "organization_entitlements", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "organization_entitlements_subscription_mode_check": { + "name": "organization_entitlements_subscription_mode_check", + "value": "\"organization_entitlements\".\"subscription_mode\" IN ('hosted', 'byok')" + }, + "organization_entitlements_status_check": { + "name": "organization_entitlements_status_check", + "value": "\"organization_entitlements\".\"status\" IN ('active', 'trialing', 'past_due', 'suspended')" + }, + "organization_entitlements_included_usage_micros_nonnegative": { + "name": "organization_entitlements_included_usage_micros_nonnegative", + "value": "\"organization_entitlements\".\"included_usage_micros\" >= 0" + }, + "organization_entitlements_overage_cap_micros_nonnegative": { + "name": "organization_entitlements_overage_cap_micros_nonnegative", + "value": "\"organization_entitlements\".\"overage_hard_cap_micros\" IS NULL OR \"organization_entitlements\".\"overage_hard_cap_micros\" >= 0" + }, + "organization_entitlements_included_usage_nonnegative": { + "name": "organization_entitlements_included_usage_nonnegative", + "value": "\"organization_entitlements\".\"included_usage_cents\" >= 0" + }, + "organization_entitlements_overage_cap_nonnegative": { + "name": "organization_entitlements_overage_cap_nonnegative", + "value": "\"organization_entitlements\".\"overage_hard_cap_cents\" IS NULL OR \"organization_entitlements\".\"overage_hard_cap_cents\" >= 0" + }, + "organization_entitlements_updated_by_nonempty": { + "name": "organization_entitlements_updated_by_nonempty", + "value": "length(btrim(\"organization_entitlements\".\"updated_by\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.organization_notification_preferences": { + "name": "organization_notification_preferences", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "billing_summary_email": { + "name": "billing_summary_email", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "service_summary_email": { + "name": "service_summary_email", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "organization_notification_preferences_org_id_organizations_id_fk": { + "name": "organization_notification_preferences_org_id_organizations_id_fk", + "tableFrom": "organization_notification_preferences", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_setting_events": { + "name": "organization_setting_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "organization_setting_events_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "setting": { + "name": "setting", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'dashboard'" + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organization_setting_events_org_time_idx": { + "name": "organization_setting_events_org_time_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "organization_setting_events_org_id_organizations_id_fk": { + "name": "organization_setting_events_org_id_organizations_id_fk", + "tableFrom": "organization_setting_events", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "organization_setting_events_actor_user_id_users_id_fk": { + "name": "organization_setting_events_actor_user_id_users_id_fk", + "tableFrom": "organization_setting_events", + "columnsFrom": [ + "actor_user_id" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "organization_setting_events_setting_check": { + "name": "organization_setting_events_setting_check", + "value": "\"organization_setting_events\".\"setting\" IN ('gate_enabled', 'billing_summary_email', 'service_summary_email')" + }, + "organization_setting_events_value_check": { + "name": "organization_setting_events_value_check", + "value": "\"organization_setting_events\".\"value\" IN ('enabled', 'disabled', 'advisory')" + }, + "organization_setting_events_source_check": { + "name": "organization_setting_events_source_check", + "value": "\"organization_setting_events\".\"source\" IN ('dashboard')" + } + }, + "isRLSEnabled": false + }, + "public.organizations": { + "name": "organizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "organizations_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "github_org_id": { + "name": "github_org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "plan": { + "name": "plan", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'beta'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organizations_github_org_id_idx": { + "name": "organizations_github_org_id_idx", + "columns": [ + { + "expression": "github_org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "organizations_slug_unique": { + "name": "organizations_slug_unique", + "columns": [ + "slug" + ], + "nullsNotDistinct": false + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.private_monitor_incidents": { + "name": "private_monitor_incidents", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "group": { + "name": "group", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "detail": { + "name": "detail", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "occurrence_count": { + "name": "occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "first_detected_at": { + "name": "first_detected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_detected_at": { + "name": "last_detected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "pending_notification_key": { + "name": "pending_notification_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pending_notification_kind": { + "name": "pending_notification_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "notification_attempts": { + "name": "notification_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "notification_available_at": { + "name": "notification_available_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "notification_lease_owner": { + "name": "notification_lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "notification_lease_expires_at": { + "name": "notification_lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_notified_at": { + "name": "last_notified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_notification_error": { + "name": "last_notification_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "private_monitor_incidents_state_updated_idx": { + "name": "private_monitor_incidents_state_updated_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "private_monitor_incidents_notification_idx": { + "name": "private_monitor_incidents_notification_idx", + "columns": [ + { + "expression": "notification_available_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "notification_lease_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "where": "\"private_monitor_incidents\".\"pending_notification_key\" IS NOT NULL", + "concurrently": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "private_monitor_incidents_state_check": { + "name": "private_monitor_incidents_state_check", + "value": "\"private_monitor_incidents\".\"state\" IN ('open', 'resolved')" + }, + "private_monitor_incidents_severity_check": { + "name": "private_monitor_incidents_severity_check", + "value": "\"private_monitor_incidents\".\"severity\" IN ('warning', 'critical')" + }, + "private_monitor_incidents_notification_kind_check": { + "name": "private_monitor_incidents_notification_kind_check", + "value": "\"private_monitor_incidents\".\"pending_notification_kind\" IS NULL OR \"private_monitor_incidents\".\"pending_notification_kind\" IN ('opened', 'reminder', 'resolved')" + }, + "private_monitor_incidents_notification_pair_check": { + "name": "private_monitor_incidents_notification_pair_check", + "value": "(\"private_monitor_incidents\".\"pending_notification_key\" IS NULL) = (\"private_monitor_incidents\".\"pending_notification_kind\" IS NULL)" + }, + "private_monitor_incidents_occurrence_count_check": { + "name": "private_monitor_incidents_occurrence_count_check", + "value": "\"private_monitor_incidents\".\"occurrence_count\" > 0 AND \"private_monitor_incidents\".\"notification_attempts\" >= 0 AND \"private_monitor_incidents\".\"notification_attempts\" <= 5" + }, + "private_monitor_incidents_text_nonempty": { + "name": "private_monitor_incidents_text_nonempty", + "value": "length(btrim(\"private_monitor_incidents\".\"key\")) > 0 AND length(btrim(\"private_monitor_incidents\".\"group\")) > 0 AND length(btrim(\"private_monitor_incidents\".\"summary\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.private_monitor_runs": { + "name": "private_monitor_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "private_monitor_runs_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "scheduled_for": { + "name": "scheduled_for", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "check_count": { + "name": "check_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "failure_count": { + "name": "failure_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "private_monitor_runs_scheduled_idx": { + "name": "private_monitor_runs_scheduled_idx", + "columns": [ + { + "expression": "scheduled_for", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "private_monitor_runs_started_idx": { + "name": "private_monitor_runs_started_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "private_monitor_runs_status_check": { + "name": "private_monitor_runs_status_check", + "value": "\"private_monitor_runs\".\"status\" IN ('running', 'completed', 'failed')" + }, + "private_monitor_runs_counts_check": { + "name": "private_monitor_runs_counts_check", + "value": "\"private_monitor_runs\".\"check_count\" >= 0 AND \"private_monitor_runs\".\"failure_count\" >= 0 AND \"private_monitor_runs\".\"failure_count\" <= \"private_monitor_runs\".\"check_count\"" + }, + "private_monitor_runs_owner_nonempty": { + "name": "private_monitor_runs_owner_nonempty", + "value": "length(btrim(\"private_monitor_runs\".\"owner\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.private_monitor_state": { + "name": "private_monitor_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_started_at": { + "name": "last_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_completed_at": { + "name": "last_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "private_monitor_state_singleton_check": { + "name": "private_monitor_state_singleton_check", + "value": "\"private_monitor_state\".\"id\" = 1" + } + }, + "isRLSEnabled": false + }, + "public.private_worker_rehearsals": { + "name": "private_worker_rehearsals", + "schema": "", + "columns": { + "nonce": { + "name": "nonce", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'armed'" + }, + "operator_github_id": { + "name": "operator_github_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "review_id": { + "name": "review_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "org_slug": { + "name": "org_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repo_full_name": { + "name": "repo_full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pr_number": { + "name": "pr_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "head_sha": { + "name": "head_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "review_public_id": { + "name": "review_public_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "armed_at": { + "name": "armed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "interrupted_worker_instance": { + "name": "interrupted_worker_instance", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "replacement_worker_instance": { + "name": "replacement_worker_instance", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "replacement_observed_at": { + "name": "replacement_observed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "before_review_count": { + "name": "before_review_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "before_usage_count": { + "name": "before_usage_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "before_check_count": { + "name": "before_check_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "before_publication_count": { + "name": "before_publication_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "after_review_count": { + "name": "after_review_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "after_usage_count": { + "name": "after_usage_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "after_check_count": { + "name": "after_check_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "after_publication_count": { + "name": "after_publication_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "private_worker_rehearsals_review_idx": { + "name": "private_worker_rehearsals_review_idx", + "columns": [ + { + "expression": "review_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "private_worker_rehearsals_job_idx": { + "name": "private_worker_rehearsals_job_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "private_worker_rehearsals_state_idx": { + "name": "private_worker_rehearsals_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "private_worker_rehearsals_org_id_organizations_id_fk": { + "name": "private_worker_rehearsals_org_id_organizations_id_fk", + "tableFrom": "private_worker_rehearsals", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "restrict" + }, + "private_worker_rehearsals_repository_id_repositories_id_fk": { + "name": "private_worker_rehearsals_repository_id_repositories_id_fk", + "tableFrom": "private_worker_rehearsals", + "columnsFrom": [ + "repository_id" + ], + "tableTo": "repositories", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "restrict" + }, + "private_worker_rehearsals_review_id_reviews_id_fk": { + "name": "private_worker_rehearsals_review_id_reviews_id_fk", + "tableFrom": "private_worker_rehearsals", + "columnsFrom": [ + "review_id" + ], + "tableTo": "reviews", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "restrict" + }, + "private_worker_rehearsals_job_id_jobs_id_fk": { + "name": "private_worker_rehearsals_job_id_jobs_id_fk", + "tableFrom": "private_worker_rehearsals", + "columnsFrom": [ + "job_id" + ], + "tableTo": "jobs", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "restrict" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "private_worker_rehearsals_state_check": { + "name": "private_worker_rehearsals_state_check", + "value": "\"private_worker_rehearsals\".\"state\" IN ('armed', 'awaiting_replacement', 'replacement_verified', 'completed', 'expired', 'failed')" + }, + "private_worker_rehearsals_identity_check": { + "name": "private_worker_rehearsals_identity_check", + "value": "length(btrim(\"private_worker_rehearsals\".\"org_slug\")) > 0 AND length(btrim(\"private_worker_rehearsals\".\"repo_full_name\")) > 0 AND \"private_worker_rehearsals\".\"pr_number\" > 0 AND \"private_worker_rehearsals\".\"head_sha\" ~ '^[0-9a-f]{40}$'" + }, + "private_worker_rehearsals_arming_window_check": { + "name": "private_worker_rehearsals_arming_window_check", + "value": "\"private_worker_rehearsals\".\"expires_at\" > \"private_worker_rehearsals\".\"armed_at\" AND \"private_worker_rehearsals\".\"expires_at\" <= \"private_worker_rehearsals\".\"armed_at\" + interval '10 minutes'" + }, + "private_worker_rehearsals_before_counts_check": { + "name": "private_worker_rehearsals_before_counts_check", + "value": "(\"private_worker_rehearsals\".\"before_review_count\" IS NULL AND \"private_worker_rehearsals\".\"before_usage_count\" IS NULL AND \"private_worker_rehearsals\".\"before_check_count\" IS NULL AND \"private_worker_rehearsals\".\"before_publication_count\" IS NULL) OR (\"private_worker_rehearsals\".\"before_review_count\" >= 0 AND \"private_worker_rehearsals\".\"before_usage_count\" >= 0 AND \"private_worker_rehearsals\".\"before_check_count\" >= 0 AND \"private_worker_rehearsals\".\"before_publication_count\" >= 0)" + }, + "private_worker_rehearsals_after_counts_check": { + "name": "private_worker_rehearsals_after_counts_check", + "value": "(\"private_worker_rehearsals\".\"after_review_count\" IS NULL AND \"private_worker_rehearsals\".\"after_usage_count\" IS NULL AND \"private_worker_rehearsals\".\"after_check_count\" IS NULL AND \"private_worker_rehearsals\".\"after_publication_count\" IS NULL) OR (\"private_worker_rehearsals\".\"after_review_count\" >= 0 AND \"private_worker_rehearsals\".\"after_usage_count\" >= 0 AND \"private_worker_rehearsals\".\"after_check_count\" >= 0 AND \"private_worker_rehearsals\".\"after_publication_count\" >= 0)" + }, + "private_worker_rehearsals_replacement_pair_check": { + "name": "private_worker_rehearsals_replacement_pair_check", + "value": "(\"private_worker_rehearsals\".\"replacement_worker_instance\" IS NULL) = (\"private_worker_rehearsals\".\"replacement_observed_at\" IS NULL)" + }, + "private_worker_rehearsals_consumed_state_check": { + "name": "private_worker_rehearsals_consumed_state_check", + "value": "(\"private_worker_rehearsals\".\"state\" IN ('armed', 'expired') AND \"private_worker_rehearsals\".\"consumed_at\" IS NULL AND \"private_worker_rehearsals\".\"interrupted_worker_instance\" IS NULL AND \"private_worker_rehearsals\".\"before_review_count\" IS NULL) OR (\"private_worker_rehearsals\".\"state\" IN ('awaiting_replacement', 'replacement_verified', 'completed', 'failed') AND \"private_worker_rehearsals\".\"consumed_at\" IS NOT NULL AND \"private_worker_rehearsals\".\"interrupted_worker_instance\" IS NOT NULL AND \"private_worker_rehearsals\".\"before_review_count\" IS NOT NULL)" + }, + "private_worker_rehearsals_replacement_state_check": { + "name": "private_worker_rehearsals_replacement_state_check", + "value": "(\"private_worker_rehearsals\".\"state\" IN ('armed', 'awaiting_replacement', 'expired') AND \"private_worker_rehearsals\".\"replacement_worker_instance\" IS NULL) OR (\"private_worker_rehearsals\".\"state\" IN ('replacement_verified', 'completed') AND \"private_worker_rehearsals\".\"replacement_worker_instance\" IS NOT NULL) OR \"private_worker_rehearsals\".\"state\" = 'failed'" + }, + "private_worker_rehearsals_completion_state_check": { + "name": "private_worker_rehearsals_completion_state_check", + "value": "(\"private_worker_rehearsals\".\"state\" = 'completed' AND \"private_worker_rehearsals\".\"after_review_count\" IS NOT NULL AND \"private_worker_rehearsals\".\"completed_at\" IS NOT NULL AND \"private_worker_rehearsals\".\"failure_reason\" IS NULL) OR (\"private_worker_rehearsals\".\"state\" IN ('expired', 'failed') AND \"private_worker_rehearsals\".\"after_review_count\" IS NULL AND \"private_worker_rehearsals\".\"completed_at\" IS NOT NULL AND \"private_worker_rehearsals\".\"failure_reason\" IS NOT NULL) OR (\"private_worker_rehearsals\".\"state\" IN ('armed', 'awaiting_replacement', 'replacement_verified') AND \"private_worker_rehearsals\".\"after_review_count\" IS NULL AND \"private_worker_rehearsals\".\"completed_at\" IS NULL AND \"private_worker_rehearsals\".\"failure_reason\" IS NULL)" + } + }, + "isRLSEnabled": false + }, + "public.release_steps": { + "name": "release_steps", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.repo_config_probes": { + "name": "repo_config_probes", + "schema": "", + "columns": { + "repository_id": { + "name": "repository_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "probed_at": { + "name": "probed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "ok": { + "name": "ok", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "files": { + "name": "files", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "repo_config_probes_repository_id_repositories_id_fk": { + "name": "repo_config_probes_repository_id_repositories_id_fk", + "tableFrom": "repo_config_probes", + "columnsFrom": [ + "repository_id" + ], + "tableTo": "repositories", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.repositories": { + "name": "repositories", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "repositories_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "installation_id": { + "name": "installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "github_repo_id": { + "name": "github_repo_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "full_name": { + "name": "full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private": { + "name": "private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "repositories_installation_id_installations_id_fk": { + "name": "repositories_installation_id_installations_id_fk", + "tableFrom": "repositories", + "columnsFrom": [ + "installation_id" + ], + "tableTo": "installations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "repositories_github_repo_id_unique": { + "name": "repositories_github_repo_id_unique", + "columns": [ + "github_repo_id" + ], + "nullsNotDistinct": false + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.repository_enablement_events": { + "name": "repository_enablement_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "repository_enablement_events_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "github_repo_id": { + "name": "github_repo_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "repository_full_name": { + "name": "repository_full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repository_private": { + "name": "repository_private", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'dashboard'" + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "repository_enablement_events_org_time_idx": { + "name": "repository_enablement_events_org_time_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "repository_enablement_events_repo_time_idx": { + "name": "repository_enablement_events_repo_time_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "repository_enablement_events_org_id_organizations_id_fk": { + "name": "repository_enablement_events_org_id_organizations_id_fk", + "tableFrom": "repository_enablement_events", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "repository_enablement_events_repository_id_repositories_id_fk": { + "name": "repository_enablement_events_repository_id_repositories_id_fk", + "tableFrom": "repository_enablement_events", + "columnsFrom": [ + "repository_id" + ], + "tableTo": "repositories", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + }, + "repository_enablement_events_actor_user_id_users_id_fk": { + "name": "repository_enablement_events_actor_user_id_users_id_fk", + "tableFrom": "repository_enablement_events", + "columnsFrom": [ + "actor_user_id" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "repository_enablement_events_action_check": { + "name": "repository_enablement_events_action_check", + "value": "\"repository_enablement_events\".\"action\" IN ('enable', 'disable')" + }, + "repository_enablement_events_source_check": { + "name": "repository_enablement_events_source_check", + "value": "\"repository_enablement_events\".\"source\" IN ('dashboard', 'github_installation', 'github_pull_request', 'github_transfer', 'github_uninstall', 'migration_baseline')" + } + }, + "isRLSEnabled": false + }, + "public.repository_gate_enforcement": { + "name": "repository_gate_enforcement", + "schema": "", + "columns": { + "repository_id": { + "name": "repository_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "branch_protection": { + "name": "branch_protection", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "evidence": { + "name": "evidence", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_successful_at": { + "name": "last_successful_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "repository_gate_enforcement_status_checked_idx": { + "name": "repository_gate_enforcement_status_checked_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "checked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "repository_gate_enforcement_repository_id_repositories_id_fk": { + "name": "repository_gate_enforcement_repository_id_repositories_id_fk", + "tableFrom": "repository_gate_enforcement", + "columnsFrom": [ + "repository_id" + ], + "tableTo": "repositories", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "repository_gate_enforcement_status_check": { + "name": "repository_gate_enforcement_status_check", + "value": "\"repository_gate_enforcement\".\"status\" IN ('required', 'not_required', 'unknown')" + }, + "repository_gate_enforcement_branch_protection_check": { + "name": "repository_gate_enforcement_branch_protection_check", + "value": "\"repository_gate_enforcement\".\"branch_protection\" IN ('protected', 'unprotected', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.respond_deliveries": { + "name": "respond_deliveries", + "schema": "", + "columns": { + "job_id": { + "name": "job_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "reservation_id": { + "name": "reservation_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "source_org_id": { + "name": "source_org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_installation_id": { + "name": "source_installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_github_installation_id": { + "name": "source_github_installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_github_repo_id": { + "name": "source_github_repo_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "repo_full_name": { + "name": "repo_full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_number": { + "name": "issue_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "is_pr": { + "name": "is_pr", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "source_head_sha": { + "name": "source_head_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "marker_nonce": { + "name": "marker_nonce", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "reply_to_review_comment_id": { + "name": "reply_to_review_comment_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "publication_identity_state": { + "name": "publication_identity_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'complete'" + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'prepared'" + }, + "delivery_lease_expires_at": { + "name": "delivery_lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "github_comment_id": { + "name": "github_comment_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "publication_lease_id": { + "name": "publication_lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "publication_lease_expires_at": { + "name": "publication_lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "cancelled_at": { + "name": "cancelled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "delivered_at": { + "name": "delivered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "respond_deliveries_pending_idx": { + "name": "respond_deliveries_pending_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "delivery_lease_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "respond_deliveries_job_id_jobs_id_fk": { + "name": "respond_deliveries_job_id_jobs_id_fk", + "tableFrom": "respond_deliveries", + "columnsFrom": [ + "job_id" + ], + "tableTo": "jobs", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "respond_deliveries_repository_id_repositories_id_fk": { + "name": "respond_deliveries_repository_id_repositories_id_fk", + "tableFrom": "respond_deliveries", + "columnsFrom": [ + "repository_id" + ], + "tableTo": "repositories", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "respond_deliveries_reservation_id_hosted_usage_reservations_id_fk": { + "name": "respond_deliveries_reservation_id_hosted_usage_reservations_id_fk", + "tableFrom": "respond_deliveries", + "columnsFrom": [ + "reservation_id" + ], + "tableTo": "hosted_usage_reservations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "respond_deliveries_state_check": { + "name": "respond_deliveries_state_check", + "value": "\"respond_deliveries\".\"state\" IN ('prepared', 'delivering', 'delivered', 'cancelled')" + }, + "respond_deliveries_issue_number_positive": { + "name": "respond_deliveries_issue_number_positive", + "value": "\"respond_deliveries\".\"issue_number\" > 0" + }, + "respond_deliveries_body_nonempty": { + "name": "respond_deliveries_body_nonempty", + "value": "length(btrim(\"respond_deliveries\".\"body\")) > 0" + }, + "respond_deliveries_publication_identity_state_check": { + "name": "respond_deliveries_publication_identity_state_check", + "value": "\"respond_deliveries\".\"publication_identity_state\" IN ('complete', 'legacy_delivered', 'cancelled_incomplete')" + }, + "respond_deliveries_publication_identity_check": { + "name": "respond_deliveries_publication_identity_check", + "value": "(\n \"respond_deliveries\".\"source_org_id\" IS NOT NULL\n AND \"respond_deliveries\".\"source_installation_id\" IS NOT NULL\n AND \"respond_deliveries\".\"source_github_installation_id\" IS NOT NULL\n AND \"respond_deliveries\".\"source_github_repo_id\" IS NOT NULL\n AND (NOT \"respond_deliveries\".\"is_pr\" OR \"respond_deliveries\".\"source_head_sha\" IS NOT NULL)\n )" + }, + "respond_deliveries_publication_identity_state_matches_row_check": { + "name": "respond_deliveries_publication_identity_state_matches_row_check", + "value": "(\n \"respond_deliveries\".\"publication_identity_state\" = 'complete'\n OR (\"respond_deliveries\".\"publication_identity_state\" = 'legacy_delivered' AND \"respond_deliveries\".\"state\" = 'delivered')\n OR (\"respond_deliveries\".\"publication_identity_state\" = 'cancelled_incomplete' AND \"respond_deliveries\".\"state\" = 'cancelled')\n )" + } + }, + "isRLSEnabled": false + }, + "public.review_logs": { + "name": "review_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "review_logs_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "review_id": { + "name": "review_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "at": { + "name": "at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "line": { + "name": "line", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "review_logs_review_seq_idx": { + "name": "review_logs_review_seq_idx", + "columns": [ + { + "expression": "review_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": { + "review_logs_review_id_reviews_id_fk": { + "name": "review_logs_review_id_reviews_id_fk", + "tableFrom": "review_logs", + "columnsFrom": [ + "review_id" + ], + "tableTo": "reviews", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.review_publication_receipts": { + "name": "review_publication_receipts", + "schema": "", + "columns": { + "review_id": { + "name": "review_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "receipt_version": { + "name": "receipt_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "receipt_id": { + "name": "receipt_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "publication_channel": { + "name": "publication_channel", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "github_review_id": { + "name": "github_review_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "observed_at": { + "name": "observed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "review_publication_receipts_review_id_reviews_id_fk": { + "name": "review_publication_receipts_review_id_reviews_id_fk", + "tableFrom": "review_publication_receipts", + "columnsFrom": [ + "review_id" + ], + "tableTo": "reviews", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "review_publication_receipts_identity_check": { + "name": "review_publication_receipts_identity_check", + "value": "(\"review_publication_receipts\".\"receipt_version\" IS NULL AND \"review_publication_receipts\".\"receipt_id\" IS NULL AND \"review_publication_receipts\".\"publication_channel\" IS NULL) OR (\"review_publication_receipts\".\"receipt_version\" = 1 AND length(btrim(\"review_publication_receipts\".\"receipt_id\")) BETWEEN 1 AND 200 AND (\"review_publication_receipts\".\"publication_channel\" IS NULL OR \"review_publication_receipts\".\"publication_channel\" = 'reviewComments')) OR (\"review_publication_receipts\".\"receipt_version\" = 2 AND length(btrim(\"review_publication_receipts\".\"receipt_id\")) BETWEEN 1 AND 200 AND \"review_publication_receipts\".\"publication_channel\" IS NOT NULL AND \"review_publication_receipts\".\"publication_channel\" IN ('reviewComments', 'checkAnnotations'))" + }, + "review_publication_receipts_github_review_id_check": { + "name": "review_publication_receipts_github_review_id_check", + "value": "\"review_publication_receipts\".\"github_review_id\" IS NULL OR \"review_publication_receipts\".\"github_review_id\" ~ '^[1-9][0-9]{0,19}$'" + } + }, + "isRLSEnabled": false + }, + "public.reviews": { + "name": "reviews", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "reviews_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "public_id": { + "name": "public_id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "repository_id": { + "name": "repository_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "source_org_id": { + "name": "source_org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_installation_id": { + "name": "source_installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_github_installation_id": { + "name": "source_github_installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_github_repo_id": { + "name": "source_github_repo_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_repo_full_name": { + "name": "source_repo_full_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pr_number": { + "name": "pr_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "author_github_id": { + "name": "author_github_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "author_login": { + "name": "author_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "head_sha": { + "name": "head_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_sha": { + "name": "base_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "since_sha": { + "name": "since_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "trigger_source": { + "name": "trigger_source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "trigger_context": { + "name": "trigger_context", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "review_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "envelope": { + "name": "envelope", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "config_files": { + "name": "config_files", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "config_provenance": { + "name": "config_provenance", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "silent": { + "name": "silent", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "engine_gate_failing": { + "name": "engine_gate_failing", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "gate_failing": { + "name": "gate_failing", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "advisory_check_run_id": { + "name": "advisory_check_run_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "gate_check_run_id": { + "name": "gate_check_run_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "gate_sync_lease_id": { + "name": "gate_sync_lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "gate_sync_lease_expires_at": { + "name": "gate_sync_lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "publication_lifecycle_reconciled_at": { + "name": "publication_lifecycle_reconciled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "publication_lifecycle_required_at": { + "name": "publication_lifecycle_required_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "queued_at": { + "name": "queued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "reviews_public_id_idx": { + "name": "reviews_public_id_idx", + "columns": [ + { + "expression": "public_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "with": {}, + "method": "btree", + "concurrently": false + }, + "reviews_repo_pr_idx": { + "name": "reviews_repo_pr_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "pr_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "reviews_status_idx": { + "name": "reviews_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "reviews_running_started_at_idx": { + "name": "reviews_running_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "where": "\"reviews\".\"status\" = 'running'", + "concurrently": false + }, + "reviews_publication_lifecycle_pending_idx": { + "name": "reviews_publication_lifecycle_pending_idx", + "columns": [ + { + "expression": "finished_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "where": "\"reviews\".\"status\" = 'completed' AND \"reviews\".\"publication_lifecycle_required_at\" IS NOT NULL AND \"reviews\".\"publication_lifecycle_reconciled_at\" IS NULL", + "concurrently": false + } + }, + "foreignKeys": { + "reviews_repository_id_repositories_id_fk": { + "name": "reviews_repository_id_repositories_id_fk", + "tableFrom": "reviews", + "columnsFrom": [ + "repository_id" + ], + "tableTo": "repositories", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "reviews_trigger_source_check": { + "name": "reviews_trigger_source_check", + "value": "\"reviews\".\"trigger_source\" IN ('unknown', 'automatic_pull_request', 'requested_review', 'github_check_rerun')" + }, + "reviews_trigger_context_check": { + "name": "reviews_trigger_context_check", + "value": "(\"reviews\".\"trigger_source\" = 'unknown' AND (\"reviews\".\"trigger_context\" IS NULL OR \"reviews\".\"trigger_context\" = '{\"source\":\"unknown\"}'::jsonb)) OR (\"reviews\".\"trigger_source\" <> 'unknown' AND \"reviews\".\"trigger_context\" IS NOT NULL AND jsonb_typeof(\"reviews\".\"trigger_context\") = 'object' AND \"reviews\".\"trigger_context\" - ARRAY['source', 'webhookDeliveryId', 'webhookEvent', 'webhookAction', 'sourceCommentId', 'sourceUrl', 'requestedByGithubId', 'requestedByLogin', 'checkName']::text[] = '{}'::jsonb AND \"reviews\".\"trigger_context\"->>'source' = \"reviews\".\"trigger_source\" AND jsonb_typeof(\"reviews\".\"trigger_context\"->'webhookDeliveryId') = 'string' AND COALESCE(length(btrim(\"reviews\".\"trigger_context\"->>'webhookDeliveryId')), 0) > 0 AND length(\"reviews\".\"trigger_context\"->>'webhookDeliveryId') <= 200 AND ((\"reviews\".\"trigger_source\" = 'automatic_pull_request' AND \"reviews\".\"trigger_context\"->>'webhookEvent' = 'pull_request') OR (\"reviews\".\"trigger_source\" = 'requested_review' AND \"reviews\".\"trigger_context\"->>'webhookEvent' IN ('issue_comment', 'pull_request_review_comment')) OR (\"reviews\".\"trigger_source\" = 'github_check_rerun' AND \"reviews\".\"trigger_context\"->>'webhookEvent' IN ('check_run', 'check_suite'))) AND (NOT \"reviews\".\"trigger_context\" ? 'webhookAction' OR (jsonb_typeof(\"reviews\".\"trigger_context\"->'webhookAction') = 'string' AND length(\"reviews\".\"trigger_context\"->>'webhookAction') <= 100)) AND (NOT \"reviews\".\"trigger_context\" ? 'sourceCommentId' OR (jsonb_typeof(\"reviews\".\"trigger_context\"->'sourceCommentId') = 'number' AND (\"reviews\".\"trigger_context\"->>'sourceCommentId')::numeric = trunc((\"reviews\".\"trigger_context\"->>'sourceCommentId')::numeric) AND (\"reviews\".\"trigger_context\"->>'sourceCommentId')::numeric BETWEEN 1 AND 9007199254740991)) AND (NOT \"reviews\".\"trigger_context\" ? 'sourceUrl' OR (jsonb_typeof(\"reviews\".\"trigger_context\"->'sourceUrl') = 'string' AND length(\"reviews\".\"trigger_context\"->>'sourceUrl') <= 2048 AND \"reviews\".\"trigger_context\"->>'sourceUrl' ~* '^https://github[.]com([/?#]|$)')) AND (NOT \"reviews\".\"trigger_context\" ? 'requestedByGithubId' OR (jsonb_typeof(\"reviews\".\"trigger_context\"->'requestedByGithubId') = 'number' AND (\"reviews\".\"trigger_context\"->>'requestedByGithubId')::numeric = trunc((\"reviews\".\"trigger_context\"->>'requestedByGithubId')::numeric) AND (\"reviews\".\"trigger_context\"->>'requestedByGithubId')::numeric BETWEEN 1 AND 9007199254740991)) AND (NOT \"reviews\".\"trigger_context\" ? 'requestedByLogin' OR (jsonb_typeof(\"reviews\".\"trigger_context\"->'requestedByLogin') = 'string' AND length(\"reviews\".\"trigger_context\"->>'requestedByLogin') <= 100)) AND (NOT \"reviews\".\"trigger_context\" ? 'checkName' OR (jsonb_typeof(\"reviews\".\"trigger_context\"->'checkName') = 'string' AND length(\"reviews\".\"trigger_context\"->>'checkName') <= 200)))" + } + }, + "isRLSEnabled": false + }, + "public.self_service_trial_grants": { + "name": "self_service_trial_grants", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "initiated_by_github_id": { + "name": "initiated_by_github_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "requested_mode": { + "name": "requested_mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_mode": { + "name": "granted_mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "self_service_trial_grants_actor_created_idx": { + "name": "self_service_trial_grants_actor_created_idx", + "columns": [ + { + "expression": "initiated_by_github_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "self_service_trial_grants_requested_mode_check": { + "name": "self_service_trial_grants_requested_mode_check", + "value": "\"self_service_trial_grants\".\"requested_mode\" IN ('hosted', 'byok')" + }, + "self_service_trial_grants_granted_mode_check": { + "name": "self_service_trial_grants_granted_mode_check", + "value": "\"self_service_trial_grants\".\"granted_mode\" IN ('hosted', 'byok')" + } + }, + "isRLSEnabled": false + }, + "public.service_heartbeats": { + "name": "service_heartbeats", + "schema": "", + "columns": { + "component": { + "name": "component", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "observed_at": { + "name": "observed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "service_heartbeats_component_check": { + "name": "service_heartbeats_component_check", + "value": "\"service_heartbeats\".\"component\" IN ('worker', 'monitor', 'monitor-heartbeat-delivery')" + }, + "service_heartbeats_instance_nonempty": { + "name": "service_heartbeats_instance_nonempty", + "value": "length(btrim(\"service_heartbeats\".\"instance_id\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "github_access_token_ciphertext": { + "name": "github_access_token_ciphertext", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "membership_checked_at": { + "name": "membership_checked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "membership_check_available_at": { + "name": "membership_check_available_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "columnsFrom": [ + "user_id" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_events": { + "name": "usage_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "usage_events_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "repository_id": { + "name": "repository_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "review_id": { + "name": "review_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "trigger_source": { + "name": "trigger_source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "prompt_tokens": { + "name": "prompt_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "completion_tokens": { + "name": "completion_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "model_used": { + "name": "model_used", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cost_micros": { + "name": "cost_micros", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cost_cents": { + "name": "cost_cents", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "billing_scope": { + "name": "billing_scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "usage_events_org_id_organizations_id_fk": { + "name": "usage_events_org_id_organizations_id_fk", + "tableFrom": "usage_events", + "columnsFrom": [ + "org_id" + ], + "tableTo": "organizations", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + }, + "usage_events_repository_id_repositories_id_fk": { + "name": "usage_events_repository_id_repositories_id_fk", + "tableFrom": "usage_events", + "columnsFrom": [ + "repository_id" + ], + "tableTo": "repositories", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + }, + "usage_events_review_id_reviews_id_fk": { + "name": "usage_events_review_id_reviews_id_fk", + "tableFrom": "usage_events", + "columnsFrom": [ + "review_id" + ], + "tableTo": "reviews", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "usage_events_cost_micros_nonnegative": { + "name": "usage_events_cost_micros_nonnegative", + "value": "\"usage_events\".\"cost_micros\" IS NULL OR \"usage_events\".\"cost_micros\" >= 0" + }, + "usage_events_cost_cents_nonnegative": { + "name": "usage_events_cost_cents_nonnegative", + "value": "\"usage_events\".\"cost_cents\" IS NULL OR \"usage_events\".\"cost_cents\" >= 0" + }, + "usage_events_billing_scope_check": { + "name": "usage_events_billing_scope_check", + "value": "\"usage_events\".\"billing_scope\" IN ('analytics', 'private_hosted')" + }, + "usage_events_trigger_source_check": { + "name": "usage_events_trigger_source_check", + "value": "\"usage_events\".\"trigger_source\" IN ('unknown', 'automatic_pull_request', 'requested_review', 'github_check_rerun', 'github_mention')" + } + }, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "name": "users_id_seq", + "increment": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "startWith": "1", + "cache": "1", + "cycle": false, + "schema": "public", + "type": "always" + } + }, + "github_id": { + "name": "github_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "login": { + "name": "login", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_github_id_unique": { + "name": "users_github_id_unique", + "columns": [ + "github_id" + ], + "nullsNotDistinct": false + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook_deliveries": { + "name": "webhook_deliveries", + "schema": "", + "columns": { + "delivery_id": { + "name": "delivery_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "webhook_deliveries_completed_at_idx": { + "name": "webhook_deliveries_completed_at_idx", + "columns": [ + { + "expression": "completed_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "where": "\"webhook_deliveries\".\"completed_at\" IS NOT NULL", + "concurrently": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "webhook_deliveries_payload_completion_check": { + "name": "webhook_deliveries_payload_completion_check", + "value": "(\"webhook_deliveries\".\"payload\" IS NULL) = (\"webhook_deliveries\".\"completed_at\" IS NOT NULL)" + } + }, + "isRLSEnabled": false + } + }, + "enums": { + "public.finding_approval_role": { + "name": "finding_approval_role", + "schema": "public", + "values": [ + "member", + "admin" + ] + }, + "public.finding_approval_source": { + "name": "finding_approval_source", + "schema": "public", + "values": [ + "github", + "dashboard" + ] + }, + "public.finding_approval_verb": { + "name": "finding_approval_verb", + "schema": "public", + "values": [ + "approve", + "dismiss" + ] + }, + "public.job_status": { + "name": "job_status", + "schema": "public", + "values": [ + "queued", + "running", + "done", + "failed" + ] + }, + "public.review_status": { + "name": "review_status", + "schema": "public", + "values": [ + "queued", + "running", + "completed", + "failed", + "stale" + ] + } + }, + "schemas": {}, + "views": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 37fe42f6..11a0432c 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -407,6 +407,13 @@ "when": 1787853636499, "tag": "0058_amused_wolverine", "breakpoints": true + }, + { + "idx": 58, + "version": "7", + "when": 1787873286640, + "tag": "0059_publication_lifecycle_nonblocking_triggers", + "breakpoints": true } ] } diff --git a/tests/publication-receipt-migration.test.ts b/tests/publication-receipt-migration.test.ts index ed78bea6..0f99b4a2 100644 --- a/tests/publication-receipt-migration.test.ts +++ b/tests/publication-receipt-migration.test.ts @@ -651,6 +651,73 @@ describeDb("publication receipt migration and lifecycle", () => { }); }); + test("trigger producers park without waiting behind queued deactivation", async () => { + const transitionPool = new Pool({ connectionString: TEST_URL, max: 3 }); + const reviewId = await createRunningReview("6".repeat(40), null, 74, false); + let releaseHolder!: () => void; + const holderReleased = new Promise((resolve) => { + releaseHolder = resolve; + }); + let holderAcquired!: () => void; + const acquired = new Promise((resolve) => { + holderAcquired = resolve; + }); + const holder = withPublicationLifecycleReleaseActive( + transitionPool, + async () => { + holderAcquired(); + await holderReleased; + }, + ); + await acquired; + const deactivation = deactivatePublicationLifecycleRelease(transitionPool); + try { + await new Promise((resolve) => setTimeout(resolve, 50)); + const client = await transitionPool.connect(); + try { + await client.query("BEGIN"); + await client.query("SET LOCAL statement_timeout = '2s'"); + await client.query( + "UPDATE reviews SET envelope = $2::jsonb WHERE id = $1", + [reviewId, JSON.stringify(envelope({ head: "6".repeat(40) }))], + ); + const gate = await client.query<{ parked: boolean; dark: boolean }>( + `INSERT INTO jobs (kind, payload) + VALUES ('gate-state-sync', jsonb_build_object( + 'reviewId', $1::bigint, 'reviewPublicId', ( + SELECT public_id::text FROM reviews WHERE id = $1 + ) + )) + RETURNING + run_after = 'infinity'::timestamptz AS parked, + payload ? '_postilPublicationLifecycleDark' AS dark`, + [reviewId], + ); + await client.query("COMMIT"); + expect(gate.rows[0]).toEqual({ parked: true, dark: true }); + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw error; + } finally { + client.release(); + } + } finally { + releaseHolder(); + await holder; + await deactivation; + await transitionPool.end(); + } + const lifecycle = await pool.query<{ required: boolean }>( + `SELECT publication_lifecycle_required_at IS NOT NULL AS required + FROM reviews WHERE id = $1`, + [reviewId], + ); + expect(lifecycle.rows[0]?.required).toBe(true); + expect(await activatePublicationLifecycleRelease(pool)).toMatchObject({ + activated: true, + }); + }); + test("pull-request decision lock blocks a newer staged recurrence", async () => { const firstId = await createRunningReview("4".repeat(40), null, 73, false); const secondId = await createRunningReview( From f03d1f260bf1b5824d434f4e68cb0e19718df51a Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 23:37:42 +0000 Subject: [PATCH 11/34] Converge gates parked during activation --- ...ication_lifecycle_nonblocking_triggers.sql | 14 +++- src/lib/release-job-rollout.ts | 9 +-- tests/publication-receipt-migration.test.ts | 64 ++++++++++++++++++- 3 files changed, 74 insertions(+), 13 deletions(-) diff --git a/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql b/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql index c3d2c16f..4bb8cf16 100644 --- a/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql +++ b/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql @@ -34,26 +34,34 @@ RETURNS trigger LANGUAGE plpgsql AS $$ DECLARE + lifecycle_locked boolean := false; lifecycle_active boolean := false; BEGIN -- A failed try-lock means deactivation owns or is queued for the boundary. -- Park the job without waiting while its caller may hold narrower locks. - IF pg_try_advisory_xact_lock_shared( + lifecycle_locked := pg_try_advisory_xact_lock_shared( hashtextextended('postil:publication-lifecycle-release', 0) - ) THEN + ); + IF lifecycle_locked THEN SELECT EXISTS ( SELECT 1 FROM deployment_capabilities WHERE name = 'publication-lifecycle-fleet-active' ) INTO lifecycle_active; END IF; IF NOT lifecycle_active THEN - NEW.run_after := 'infinity'::timestamptz; + NEW.run_after := CASE + WHEN lifecycle_locked THEN 'infinity'::timestamptz + ELSE now() + interval '30 seconds' + END; NEW.payload := jsonb_set( COALESCE(NEW.payload, '{}'::jsonb), '{_postilPublicationLifecycleDark}', 'true'::jsonb, true ); + ELSE + NEW.payload := COALESCE(NEW.payload, '{}'::jsonb) + - '_postilPublicationLifecycleDark'; END IF; RETURN NEW; END; diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index 8d0f64b8..df909aa2 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -240,13 +240,8 @@ export async function activatePublicationLifecycleRelease( SET status = 'queued', locked_at = NULL, locked_by = NULL, - run_after = 'infinity'::timestamptz, - payload = jsonb_set( - payload, - ARRAY[$1]::text[], - 'true'::jsonb, - true - ), + run_after = now(), + payload = payload - $1, last_error = concat_ws( ' ', NULLIF(last_error, ''), '[release: recovered abandoned gate publisher]' diff --git a/tests/publication-receipt-migration.test.ts b/tests/publication-receipt-migration.test.ts index 0f99b4a2..75c78d07 100644 --- a/tests/publication-receipt-migration.test.ts +++ b/tests/publication-receipt-migration.test.ts @@ -681,7 +681,7 @@ describeDb("publication receipt migration and lifecycle", () => { "UPDATE reviews SET envelope = $2::jsonb WHERE id = $1", [reviewId, JSON.stringify(envelope({ head: "6".repeat(40) }))], ); - const gate = await client.query<{ parked: boolean; dark: boolean }>( + const gate = await client.query<{ deferred: boolean; dark: boolean }>( `INSERT INTO jobs (kind, payload) VALUES ('gate-state-sync', jsonb_build_object( 'reviewId', $1::bigint, 'reviewPublicId', ( @@ -689,12 +689,13 @@ describeDb("publication receipt migration and lifecycle", () => { ) )) RETURNING - run_after = 'infinity'::timestamptz AS parked, + run_after > now() + AND run_after <= now() + interval '31 seconds' AS deferred, payload ? '_postilPublicationLifecycleDark' AS dark`, [reviewId], ); await client.query("COMMIT"); - expect(gate.rows[0]).toEqual({ parked: true, dark: true }); + expect(gate.rows[0]).toEqual({ deferred: true, dark: true }); } catch (error) { await client.query("ROLLBACK").catch(() => undefined); throw error; @@ -718,6 +719,63 @@ describeDb("publication receipt migration and lifecycle", () => { }); }); + test("a gate committed after the activation sweep self-heals", async () => { + await deactivatePublicationLifecycleRelease(pool); + const activationClient = await pool.connect(); + let lateGateId = 0; + try { + await activationClient.query("BEGIN"); + await activationClient.query( + "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", + ["postil:publication-lifecycle-release"], + ); + await activationClient.query( + `INSERT INTO deployment_capabilities (name) + VALUES ('publication-lifecycle-fleet-active') + ON CONFLICT (name) DO NOTHING`, + ); + await activationClient.query( + `UPDATE jobs + SET run_after = now(), + payload = payload - '_postilPublicationLifecycleDark' + WHERE kind = 'gate-state-sync' + AND status = 'queued' + AND payload ? '_postilPublicationLifecycleDark'`, + ); + const lateGate = await pool.query<{ + id: string; + deferred: boolean; + dark: boolean; + }>( + `INSERT INTO jobs (kind, payload) + VALUES ('gate-state-sync', jsonb_build_object( + 'reviewId', 1, 'reviewPublicId', '00000000-0000-4000-8000-000000000001' + )) + RETURNING id, + run_after > now() + AND run_after <= now() + interval '31 seconds' AS deferred, + payload ? '_postilPublicationLifecycleDark' AS dark`, + ); + lateGateId = Number(lateGate.rows[0]!.id); + expect(lateGate.rows[0]).toMatchObject({ deferred: true, dark: true }); + await activationClient.query("COMMIT"); + } catch (error) { + await activationClient.query("ROLLBACK").catch(() => undefined); + throw error; + } finally { + activationClient.release(); + } + const converged = await pool.query<{ due: boolean; dark: boolean }>( + `UPDATE jobs SET run_after = now() + WHERE id = $1 + RETURNING run_after <= now() AS due, + payload ? '_postilPublicationLifecycleDark' AS dark`, + [lateGateId], + ); + expect(converged.rows[0]).toEqual({ due: true, dark: false }); + await activatePublicationLifecycleRelease(pool); + }); + test("pull-request decision lock blocks a newer staged recurrence", async () => { const firstId = await createRunningReview("4".repeat(40), null, 73, false); const secondId = await createRunningReview( From 0ed400a498a7648bc7896868ddd7fe5e746b2e01 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 23:43:45 +0000 Subject: [PATCH 12/34] Reuse clients after successful transaction rollback --- src/lib/db-transaction.ts | 3 ++- tests/private-worker-gates.test.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/db-transaction.ts b/src/lib/db-transaction.ts index aac022be..8a983fc7 100644 --- a/src/lib/db-transaction.ts +++ b/src/lib/db-transaction.ts @@ -35,8 +35,9 @@ export async function withPinnedDatabaseTransaction( } }); } catch (error) { + if (bodyFailed && error === bodyError) throw error; releaseError = databaseClientError(error, `${label} transaction failed`); - if (bodyFailed && error !== bodyError) { + if (bodyFailed) { throw new AggregateError( [databaseClientError(bodyError, `${label} operation failed`), releaseError], `${label} operation and transaction cleanup failed`, diff --git a/tests/private-worker-gates.test.ts b/tests/private-worker-gates.test.ts index cab45cbb..f2cce4e9 100644 --- a/tests/private-worker-gates.test.ts +++ b/tests/private-worker-gates.test.ts @@ -229,7 +229,7 @@ describe("private repository worker defense in depth", () => { expect(decision).not.toContain("pg_advisory_unlock("); expect(database).toContain("clientDatabase.transaction"); expect(database).toContain("client.release(releaseError)"); - expect(database).toContain("bodyFailed && error !== bodyError"); + expect(database).toContain("bodyFailed && error === bodyError"); }); test("respond honors entitlement and release activation before tokens or provider access", () => { From a097dd6bf15b52a51e65f0d69584220fd2c9e2d9 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 23:50:02 +0000 Subject: [PATCH 13/34] Clarify pinned transaction cleanup --- src/lib/db-transaction.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/lib/db-transaction.ts b/src/lib/db-transaction.ts index 8a983fc7..cfc25bc5 100644 --- a/src/lib/db-transaction.ts +++ b/src/lib/db-transaction.ts @@ -35,7 +35,12 @@ export async function withPinnedDatabaseTransaction( } }); } catch (error) { - if (bodyFailed && error === bodyError) throw error; + if (bodyFailed && error === bodyError) { + // Drizzle rethrows the callback's identical value only after ROLLBACK + // succeeds. Leave releaseError unset so pg returns this client to the + // pool; a different error means transaction cleanup was not confirmed. + throw error; + } releaseError = databaseClientError(error, `${label} transaction failed`); if (bodyFailed) { throw new AggregateError( From a7fce426a4856a15ce64eceef7bbd00effc2dd30 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 00:27:14 +0000 Subject: [PATCH 14/34] Retire stale publication lifecycle locks --- ...ication_lifecycle_nonblocking_triggers.sql | 71 +++++++++++++++++++ tests/migration-lint.test.ts | 39 ++++++++++ 2 files changed, 110 insertions(+) diff --git a/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql b/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql index 4bb8cf16..eab8f8d8 100644 --- a/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql +++ b/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql @@ -1,3 +1,74 @@ +-- Legacy transaction-pool workers can leave the session-level lifecycle lock +-- attached to an idle Supavisor backend after the logical client returns to +-- the pool. Retire only those exact idle holders before replacing the lock +-- protocol. Active sessions and every unrelated advisory lock remain untouched. +DO $$ +DECLARE + stale_pid integer; + cleanup_deadline timestamptz := clock_timestamp() + interval '30 seconds'; +BEGIN + LOOP + PERFORM pg_stat_clear_snapshot(); + stale_pid := NULL; + SELECT advisory.pid + INTO stale_pid + FROM pg_locks AS advisory + INNER JOIN pg_stat_activity AS activity ON activity.pid = advisory.pid + WHERE advisory.locktype = 'advisory' + AND advisory.granted + AND advisory.mode IN ('ShareLock', 'ExclusiveLock') + AND advisory.objsubid = 1 + AND activity.datname = current_database() + AND activity.usename = current_user + AND advisory.classid::bigint = ( + (hashtextextended('postil:publication-lifecycle-release', 0) >> 32) + & 4294967295 + ) + AND advisory.objid::bigint = ( + hashtextextended('postil:publication-lifecycle-release', 0) + & 4294967295 + ) + AND advisory.pid <> pg_backend_pid() + AND activity.application_name = 'Supavisor' + AND activity.state = 'idle' + ORDER BY advisory.pid + LIMIT 1; + + IF stale_pid IS NOT NULL THEN + PERFORM pg_terminate_backend(stale_pid); + PERFORM pg_sleep(0.05); + CONTINUE; + END IF; + + EXIT WHEN NOT EXISTS ( + SELECT 1 + FROM pg_locks AS advisory + INNER JOIN pg_stat_activity AS activity ON activity.pid = advisory.pid + WHERE advisory.locktype = 'advisory' + AND advisory.granted + AND advisory.mode IN ('ShareLock', 'ExclusiveLock') + AND advisory.objsubid = 1 + AND activity.datname = current_database() + AND activity.usename = current_user + AND advisory.classid::bigint = ( + (hashtextextended('postil:publication-lifecycle-release', 0) >> 32) + & 4294967295 + ) + AND advisory.objid::bigint = ( + hashtextextended('postil:publication-lifecycle-release', 0) + & 4294967295 + ) + AND advisory.pid <> pg_backend_pid() + AND activity.application_name = 'Supavisor' + ); + + IF clock_timestamp() >= cleanup_deadline THEN + RAISE EXCEPTION 'active legacy publication lifecycle lock did not quiesce'; + END IF; + PERFORM pg_sleep(0.1); + END LOOP; +END; +$$;--> statement-breakpoint SELECT pg_advisory_xact_lock(hashtextextended('postil:publication-lifecycle-release', 0));--> statement-breakpoint CREATE OR REPLACE FUNCTION "postil_require_publication_lifecycle"() RETURNS trigger diff --git a/tests/migration-lint.test.ts b/tests/migration-lint.test.ts index c6824df9..67774c3b 100644 --- a/tests/migration-lint.test.ts +++ b/tests/migration-lint.test.ts @@ -396,6 +396,15 @@ describe("migration lint", () => { join(import.meta.dir, "..", "drizzle", "0058_amused_wolverine.sql"), "utf8", ); + const publicationLifecycleRepairMigration = await readFile( + join( + import.meta.dir, + "..", + "drizzle", + "0059_publication_lifecycle_nonblocking_triggers.sql", + ), + "utf8", + ); expect(migration).toContain('CREATE TABLE "release_steps"'); expect(migration).not.toContain("CREATE INDEX"); @@ -413,6 +422,36 @@ describe("migration lint", () => { expect(publicationLifecycleMigration).toContain( 'UPDATE "jobs"\nSET "run_after" = \'infinity\'::timestamptz', ); + expect(publicationLifecycleRepairMigration).toContain( + "PERFORM pg_terminate_backend(stale_pid)", + ); + expect(publicationLifecycleRepairMigration).toContain( + "activity.application_name = 'Supavisor'", + ); + expect(publicationLifecycleRepairMigration).toContain( + "activity.state = 'idle'", + ); + expect(publicationLifecycleRepairMigration).toContain( + "activity.datname = current_database()", + ); + expect(publicationLifecycleRepairMigration).toContain( + "activity.usename = current_user", + ); + expect(publicationLifecycleRepairMigration).toContain( + "clock_timestamp() + interval '30 seconds'", + ); + expect(publicationLifecycleRepairMigration).toContain( + "PERFORM pg_stat_clear_snapshot()", + ); + expect(publicationLifecycleRepairMigration).toContain( + "active legacy publication lifecycle lock did not quiesce", + ); + expect(publicationLifecycleRepairMigration).toContain( + "advisory.pid <> pg_backend_pid()", + ); + expect(publicationLifecycleRepairMigration).toContain( + "hashtextextended('postil:publication-lifecycle-release', 0)", + ); expect(releaseScript).toContain( 'CREATE INDEX CONCURRENTLY IF NOT EXISTS "reviews_publication_lifecycle_pending_idx"', ); From 01bec16fede0c5727f6aba2a9a1d23de0a0cfec9 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 00:36:35 +0000 Subject: [PATCH 15/34] Bound stale lifecycle lock cleanup --- ...ication_lifecycle_nonblocking_triggers.sql | 44 +++++-------------- tests/migration-lint.test.ts | 5 ++- 2 files changed, 16 insertions(+), 33 deletions(-) diff --git a/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql b/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql index eab8f8d8..7601c66f 100644 --- a/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql +++ b/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql @@ -5,13 +5,15 @@ DO $$ DECLARE stale_pid integer; + stale_state text; cleanup_deadline timestamptz := clock_timestamp() + interval '30 seconds'; BEGIN LOOP PERFORM pg_stat_clear_snapshot(); stale_pid := NULL; - SELECT advisory.pid - INTO stale_pid + stale_state := NULL; + SELECT advisory.pid, activity.state + INTO stale_pid, stale_state FROM pg_locks AS advisory INNER JOIN pg_stat_activity AS activity ON activity.pid = advisory.pid WHERE advisory.locktype = 'advisory' @@ -30,42 +32,20 @@ BEGIN ) AND advisory.pid <> pg_backend_pid() AND activity.application_name = 'Supavisor' - AND activity.state = 'idle' - ORDER BY advisory.pid + ORDER BY (activity.state = 'idle') DESC, advisory.pid LIMIT 1; - IF stale_pid IS NOT NULL THEN - PERFORM pg_terminate_backend(stale_pid); - PERFORM pg_sleep(0.05); - CONTINUE; - END IF; - - EXIT WHEN NOT EXISTS ( - SELECT 1 - FROM pg_locks AS advisory - INNER JOIN pg_stat_activity AS activity ON activity.pid = advisory.pid - WHERE advisory.locktype = 'advisory' - AND advisory.granted - AND advisory.mode IN ('ShareLock', 'ExclusiveLock') - AND advisory.objsubid = 1 - AND activity.datname = current_database() - AND activity.usename = current_user - AND advisory.classid::bigint = ( - (hashtextextended('postil:publication-lifecycle-release', 0) >> 32) - & 4294967295 - ) - AND advisory.objid::bigint = ( - hashtextextended('postil:publication-lifecycle-release', 0) - & 4294967295 - ) - AND advisory.pid <> pg_backend_pid() - AND activity.application_name = 'Supavisor' - ); + EXIT WHEN stale_pid IS NULL; IF clock_timestamp() >= cleanup_deadline THEN RAISE EXCEPTION 'active legacy publication lifecycle lock did not quiesce'; END IF; - PERFORM pg_sleep(0.1); + IF stale_state = 'idle' THEN + PERFORM pg_terminate_backend(stale_pid); + PERFORM pg_sleep(0.05); + ELSE + PERFORM pg_sleep(0.1); + END IF; END LOOP; END; $$;--> statement-breakpoint diff --git a/tests/migration-lint.test.ts b/tests/migration-lint.test.ts index 67774c3b..3a3db521 100644 --- a/tests/migration-lint.test.ts +++ b/tests/migration-lint.test.ts @@ -429,7 +429,10 @@ describe("migration lint", () => { "activity.application_name = 'Supavisor'", ); expect(publicationLifecycleRepairMigration).toContain( - "activity.state = 'idle'", + "ORDER BY (activity.state = 'idle') DESC", + ); + expect(publicationLifecycleRepairMigration).toContain( + "IF stale_state = 'idle' THEN", ); expect(publicationLifecycleRepairMigration).toContain( "activity.datname = current_database()", From adcd6e07215b522562122e9783fce3411324e8e7 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 00:51:34 +0000 Subject: [PATCH 16/34] Fence publication lifecycle transitions --- ...ication_lifecycle_nonblocking_triggers.sql | 16 +++- package.json | 2 +- scripts/deactivate-hosted-inference.ts | 14 ++++ src/lib/github/publication-threads.ts | 4 +- src/lib/release-job-rollout.ts | 84 +++++++++++++++++-- tests/migration-lint.test.ts | 9 ++ tests/private-worker-gates.test.ts | 15 +++- tests/publication-receipt-migration.test.ts | 39 +++++++++ tests/publication-receipt.test.ts | 67 ++++++++------- tests/release-database-url.test.ts | 14 +++- 10 files changed, 217 insertions(+), 47 deletions(-) diff --git a/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql b/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql index 7601c66f..776dab57 100644 --- a/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql +++ b/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql @@ -4,11 +4,17 @@ -- protocol. Active sessions and every unrelated advisory lock remain untouched. DO $$ DECLARE + lifecycle_locked boolean := false; stale_pid integer; stale_state text; cleanup_deadline timestamptz := clock_timestamp() + interval '30 seconds'; BEGIN LOOP + lifecycle_locked := pg_try_advisory_lock( + hashtextextended('postil:publication-lifecycle-release', 0) + ); + EXIT WHEN lifecycle_locked; + PERFORM pg_stat_clear_snapshot(); stale_pid := NULL; stale_state := NULL; @@ -49,7 +55,6 @@ BEGIN END LOOP; END; $$;--> statement-breakpoint -SELECT pg_advisory_xact_lock(hashtextextended('postil:publication-lifecycle-release', 0));--> statement-breakpoint CREATE OR REPLACE FUNCTION "postil_require_publication_lifecycle"() RETURNS trigger LANGUAGE plpgsql @@ -116,4 +121,13 @@ BEGIN END IF; RETURN NEW; END; +$$;--> statement-breakpoint +DO $$ +BEGIN + IF NOT pg_advisory_unlock( + hashtextextended('postil:publication-lifecycle-release', 0) + ) THEN + RAISE EXCEPTION 'publication lifecycle migration lock was not held'; + END IF; +END; $$; diff --git a/package.json b/package.json index 7fad6d53..b77cd486 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "db:push": "drizzle-kit push", "db:migrate": "drizzle-kit migrate", "db:migrate:release": "bun run scripts/run-release-migrations.ts", - "release:prepare": "bun run db:migrate:release && bun run operational:indexes && bun run notifications:quiesce && bun run hosted:deactivate-release", + "release:prepare": "bun run hosted:deactivate-release && bun run db:migrate:release && bun run operational:indexes && bun run notifications:quiesce", "seed": "bun run scripts/seed.ts", "billing:grant-credit": "bun run scripts/grant-billing-credit.ts", "billing:set-entitlement": "bun run scripts/set-org-entitlement.ts", diff --git a/scripts/deactivate-hosted-inference.ts b/scripts/deactivate-hosted-inference.ts index 4a8bb9e2..dabc7a02 100644 --- a/scripts/deactivate-hosted-inference.ts +++ b/scripts/deactivate-hosted-inference.ts @@ -4,6 +4,7 @@ import { deactivateHostedInferenceRelease, deactivatePublicationLifecycleRelease, } from "@/lib/release-job-rollout"; +import { resolveDirectDatabaseUrl } from "./resolve-direct-database-url"; async function main(): Promise { try { @@ -12,6 +13,19 @@ async function main(): Promise { console.log("managed hosted inference preparation skipped outside a release image"); return; } + process.env.DATABASE_URL = resolveDirectDatabaseUrl({ + databaseUrl: process.env.DATABASE_URL ?? "", + directDatabaseUrl: process.env.POSTIL_DIRECT_DATABASE_URL, + }); + delete process.env.POSTIL_DIRECT_DATABASE_URL; + const schemaReady = await getPool().query<{ ready: boolean }>( + `SELECT to_regclass('public.deployment_capabilities') IS NOT NULL + AND to_regclass('public.jobs') IS NOT NULL AS ready`, + ); + if (schemaReady.rows[0]?.ready !== true) { + console.log("managed release preparation skipped until the database schema exists"); + return; + } const publicationLifecycle = await deactivatePublicationLifecycleRelease( getPool(), ); diff --git a/src/lib/github/publication-threads.ts b/src/lib/github/publication-threads.ts index bd3c6603..f462e324 100644 --- a/src/lib/github/publication-threads.ts +++ b/src/lib/github/publication-threads.ts @@ -255,7 +255,9 @@ export async function resolveGitHubReviewThreads( observation.viewerCanResolve === false && observation.state === "outdated" ) { - continue; + throw new Error( + "GitHub cannot resolve an outdated Postil review thread", + ); } if (observation.viewerCanResolve !== true) { throw new Error( diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index df909aa2..349a13d9 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -32,11 +32,86 @@ export const PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY = "publication-lifecycle-fleet-active"; const PUBLICATION_LIFECYCLE_DARK_PAYLOAD_KEY = "_postilPublicationLifecycleDark"; +const PUBLICATION_LIFECYCLE_LOCK_TIMEOUT_MS = 30_000; function databaseClientError(error: unknown, fallback: string): Error { return error instanceof Error ? error : new Error(fallback); } +async function lockPublicationLifecycleExclusive( + client: PoolClient, +): Promise { + const deadline = Date.now() + PUBLICATION_LIFECYCLE_LOCK_TIMEOUT_MS; + while (true) { + const acquired = await client.query<{ acquired: boolean }>( + "SELECT pg_try_advisory_xact_lock(hashtextextended($1, 0)) AS acquired", + [PUBLICATION_LIFECYCLE_LOCK], + ); + if (acquired.rows[0]?.acquired === true) return; + + await client.query("SELECT pg_stat_clear_snapshot()"); + const stale = await client.query<{ pid: number }>( + `SELECT advisory.pid + FROM pg_locks AS advisory + INNER JOIN pg_stat_activity AS activity ON activity.pid = advisory.pid + WHERE advisory.locktype = 'advisory' + AND advisory.granted + AND advisory.mode IN ('ShareLock', 'ExclusiveLock') + AND advisory.objsubid = 1 + AND activity.datname = current_database() + AND activity.usename = current_user + AND advisory.classid::bigint = ( + (hashtextextended($1, 0) >> 32) & 4294967295 + ) + AND advisory.objid::bigint = ( + hashtextextended($1, 0) & 4294967295 + ) + AND advisory.pid <> pg_backend_pid() + AND activity.application_name = 'Supavisor' + AND activity.state = 'idle' + ORDER BY advisory.pid + LIMIT 1`, + [PUBLICATION_LIFECYCLE_LOCK], + ); + if (stale.rows[0]) { + await client.query("SELECT pg_terminate_backend($1)", [stale.rows[0].pid]); + } + if (Date.now() >= deadline) { + throw new Error("publication lifecycle lock did not quiesce within 30 seconds"); + } + + await client.query("SAVEPOINT publication_lifecycle_lock_attempt"); + try { + // A bounded blocking request enters PostgreSQL's lock queue. Trigger + // try-locks then defer new producers instead of extending this drain. + await client.query("SET LOCAL lock_timeout = '250ms'"); + await client.query( + "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", + [PUBLICATION_LIFECYCLE_LOCK], + ); + await client.query("RELEASE SAVEPOINT publication_lifecycle_lock_attempt"); + return; + } catch (error) { + try { + await client.query("ROLLBACK TO SAVEPOINT publication_lifecycle_lock_attempt"); + await client.query("RELEASE SAVEPOINT publication_lifecycle_lock_attempt"); + } catch (cleanupError) { + throw new AggregateError( + [ + databaseClientError(error, "publication lifecycle lock attempt failed"), + databaseClientError( + cleanupError, + "publication lifecycle lock savepoint cleanup failed", + ), + ], + "publication lifecycle lock attempt and savepoint cleanup failed", + ); + } + if ((error as { code?: string }).code !== "55P03") throw error; + } + } +} + export class PublicationLifecycleReleaseDarkError extends Error { override name = "PublicationLifecycleReleaseDarkError"; @@ -92,10 +167,7 @@ export async function deactivatePublicationLifecycleRelease( let releaseError: Error | undefined; try { await client.query("BEGIN"); - await client.query( - "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", - [PUBLICATION_LIFECYCLE_LOCK], - ); + await lockPublicationLifecycleExclusive(client); const deactivated = await client.query( "DELETE FROM deployment_capabilities WHERE name = $1", [PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY], @@ -150,9 +222,7 @@ export async function activatePublicationLifecycleRelease( let releaseError: Error | undefined; try { await client.query("BEGIN"); - await client.query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [ - PUBLICATION_LIFECYCLE_LOCK, - ]); + await lockPublicationLifecycleExclusive(client); const invalid = await client.query<{ count: string }>( `SELECT count(*)::text AS count FROM reviews AS review diff --git a/tests/migration-lint.test.ts b/tests/migration-lint.test.ts index 3a3db521..a265be9d 100644 --- a/tests/migration-lint.test.ts +++ b/tests/migration-lint.test.ts @@ -443,6 +443,15 @@ describe("migration lint", () => { expect(publicationLifecycleRepairMigration).toContain( "clock_timestamp() + interval '30 seconds'", ); + expect(publicationLifecycleRepairMigration).toContain( + "lifecycle_locked := pg_try_advisory_lock(", + ); + expect(publicationLifecycleRepairMigration).toContain( + "IF NOT pg_advisory_unlock(", + ); + expect(publicationLifecycleRepairMigration).not.toContain( + "SELECT pg_advisory_xact_lock(hashtextextended('postil:publication-lifecycle-release'", + ); expect(publicationLifecycleRepairMigration).toContain( "PERFORM pg_stat_clear_snapshot()", ); diff --git a/tests/private-worker-gates.test.ts b/tests/private-worker-gates.test.ts index f2cce4e9..f0765672 100644 --- a/tests/private-worker-gates.test.ts +++ b/tests/private-worker-gates.test.ts @@ -197,6 +197,13 @@ describe("private repository worker defense in depth", () => { "src/lib/publication-lifecycle-lock.ts", "utf8", ); + const exclusiveLockStart = rollout.indexOf( + "async function lockPublicationLifecycleExclusive", + ); + const exclusiveLockEnd = rollout.indexOf( + "export class PublicationLifecycleReleaseDarkError", + exclusiveLockStart, + ); const decisionStart = decisions.indexOf( "export async function withReviewDecisionScopeLock", ); @@ -207,6 +214,7 @@ describe("private repository worker defense in depth", () => { const shared = rollout.slice(sharedStart, sharedEnd); const activation = rollout.slice(activationStart, activationEnd); + const exclusiveLock = rollout.slice(exclusiveLockStart, exclusiveLockEnd); const decision = decisions.slice(decisionStart, decisionEnd); expect(lifecycleLock).toContain("pg_advisory_xact_lock_shared"); expect(shared).toContain("withPinnedDatabaseTransaction"); @@ -215,7 +223,12 @@ describe("private repository worker defense in depth", () => { expect(shared).not.toContain("drizzle(pool"); expect(shared).not.toContain("pg_advisory_lock_shared"); expect(shared).not.toContain("pg_advisory_unlock_shared"); - expect(activation).toContain("pg_advisory_xact_lock"); + expect(activation).toContain("lockPublicationLifecycleExclusive(client)"); + expect(exclusiveLock).toContain("pg_try_advisory_xact_lock"); + expect(exclusiveLock).toContain("lock_timeout = '250ms'"); + expect(exclusiveLock).toContain("ROLLBACK TO SAVEPOINT"); + expect(exclusiveLock).toContain("pg_terminate_backend"); + expect(exclusiveLock).toContain("publication lifecycle lock did not quiesce"); expect(activation).toContain("client.release(releaseError)"); expect(activation).not.toContain('query("ROLLBACK").catch'); expect(activation).not.toContain("pg_advisory_unlock"); diff --git a/tests/publication-receipt-migration.test.ts b/tests/publication-receipt-migration.test.ts index 75c78d07..d8485858 100644 --- a/tests/publication-receipt-migration.test.ts +++ b/tests/publication-receipt-migration.test.ts @@ -719,6 +719,45 @@ describeDb("publication receipt migration and lifecycle", () => { }); }); + test("deactivation retires an idle transaction-pool session lock", async () => { + const stalePool = new Pool({ + connectionString: TEST_URL, + max: 1, + application_name: "Supavisor", + }); + const holder = await stalePool.connect(); + const holderFailure = new Promise((resolve) => { + holder.on("error", resolve); + }); + try { + await holder.query( + "SELECT pg_advisory_lock_shared(hashtextextended($1, 0))", + ["postil:publication-lifecycle-release"], + ); + + expect(await deactivatePublicationLifecycleRelease(pool)).toMatchObject({ + deactivated: true, + }); + const termination = await Promise.race([ + holderFailure, + Bun.sleep(1_000).then(() => null), + ]); + expect(termination?.message).toContain( + "terminating connection due to administrator command", + ); + } finally { + await holder + .query( + "SELECT pg_advisory_unlock_shared(hashtextextended($1, 0))", + ["postil:publication-lifecycle-release"], + ) + .catch(() => undefined); + holder.release(true); + await stalePool.end(); + await activatePublicationLifecycleRelease(pool); + } + }); + test("a gate committed after the activation sweep self-heals", async () => { await deactivatePublicationLifecycleRelease(pool); const activationClient = await pool.connect(); diff --git a/tests/publication-receipt.test.ts b/tests/publication-receipt.test.ts index ceae3ebb..6ac104f8 100644 --- a/tests/publication-receipt.test.ts +++ b/tests/publication-receipt.test.ts @@ -598,36 +598,38 @@ describe("GitHub publication thread observations", () => { })); }) as unknown as typeof fetch; + const observations = [ + { + githubCommentId: "11", + githubThreadId: "thread-11", + state: "outdated", + viewerCanResolve: true, + }, + { + githubCommentId: "12", + githubThreadId: "thread-12", + state: "inline", + viewerCanResolve: true, + }, + { + githubCommentId: "13", + githubThreadId: "thread-13", + state: "resolved", + viewerCanResolve: false, + }, + { githubCommentId: "14", state: "deleted" }, + { + githubCommentId: "15", + githubThreadId: "thread-15", + state: "outdated", + viewerCanResolve: false, + }, + ] as const; + const reconciled = await resolveGitHubReviewThreads( "token", - [ - { - githubCommentId: "11", - githubThreadId: "thread-11", - state: "outdated", - viewerCanResolve: true, - }, - { - githubCommentId: "12", - githubThreadId: "thread-12", - state: "inline", - viewerCanResolve: true, - }, - { - githubCommentId: "13", - githubThreadId: "thread-13", - state: "resolved", - viewerCanResolve: false, - }, - { githubCommentId: "14", state: "deleted" }, - { - githubCommentId: "15", - githubThreadId: "thread-15", - state: "outdated", - viewerCanResolve: false, - }, - ], - ["11", "13", "14", "15"], + observations.slice(0, -1), + ["11", "13", "14"], ); expect(requestedThreadIds).toEqual(["thread-11"]); @@ -651,13 +653,10 @@ describe("GitHub publication thread observations", () => { viewerCanResolve: false, }, { githubCommentId: "14", state: "deleted" }, - { - githubCommentId: "15", - githubThreadId: "thread-15", - state: "outdated", - viewerCanResolve: false, - }, ]); + await expect( + resolveGitHubReviewThreads("token", [...observations], ["15"]), + ).rejects.toThrow("cannot resolve an outdated Postil review thread"); }); test("fails closed when GitHub cannot resolve a still-active terminal thread", async () => { diff --git a/tests/release-database-url.test.ts b/tests/release-database-url.test.ts index 62c515a0..57cf1d95 100644 --- a/tests/release-database-url.test.ts +++ b/tests/release-database-url.test.ts @@ -44,7 +44,7 @@ describe("release database connection", () => { ).toThrow(/cannot be empty/); }); - test("binds only the migration subprocess to the direct connection", async () => { + test("binds the migration subprocess to the direct connection", async () => { const runtimeUrl = "postgresql://postgres.project@aws-0-eu-central-1.pooler.supabase.com:6543/postgres"; const directUrl = @@ -134,12 +134,22 @@ describe("release database connection", () => { scripts: Record; }; const deployWorkflow = await readFile(join(root, ".github", "workflows", "deploy.yml"), "utf8"); + const deactivationScript = await readFile( + join(root, "scripts", "deactivate-hosted-inference.ts"), + "utf8", + ); - expect(packageJson.scripts["release:prepare"]).toStartWith("bun run db:migrate:release"); + expect(packageJson.scripts["release:prepare"]).toStartWith( + "bun run hosted:deactivate-release && bun run db:migrate:release", + ); expect(packageJson.scripts["db:migrate:release"]).toBe( "bun run scripts/run-release-migrations.ts", ); expect(deployWorkflow).toContain('staged+="DATABASE_URL=${DATABASE_URL}"'); expect(deployWorkflow).not.toContain("POSTIL_DIRECT_DATABASE_URL"); + expect(deactivationScript).toContain("resolveDirectDatabaseUrl"); + expect(deactivationScript.indexOf("process.env.DATABASE_URL =")).toBeLessThan( + deactivationScript.indexOf("getPool().query"), + ); }); }); From 5cb495a2709c882d4072388710082fff6079e755 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 00:56:21 +0000 Subject: [PATCH 17/34] Clarify pre-migration lifecycle cleanup --- ...ication_lifecycle_nonblocking_triggers.sql | 9 ++--- scripts/deactivate-hosted-inference.ts | 33 ++++++++++++++----- tests/publication-receipt-migration.test.ts | 17 +++++++++- tests/release-database-url.test.ts | 1 + 4 files changed, 47 insertions(+), 13 deletions(-) diff --git a/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql b/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql index 776dab57..087e9b76 100644 --- a/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql +++ b/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql @@ -1,7 +1,8 @@ --- Legacy transaction-pool workers can leave the session-level lifecycle lock --- attached to an idle Supavisor backend after the logical client returns to --- the pool. Retire only those exact idle holders before replacing the lock --- protocol. Active sessions and every unrelated advisory lock remain untouched. +-- Transaction-pool workers can leave session state attached to an idle +-- Supavisor backend after the logical client returns to the pool. A backend +-- selected by the exact lifecycle lock has no active query, so terminating it +-- intentionally discards all leaked session state on that backend. Active +-- sessions and backends without the exact lifecycle lock remain untouched. DO $$ DECLARE lifecycle_locked boolean := false; diff --git a/scripts/deactivate-hosted-inference.ts b/scripts/deactivate-hosted-inference.ts index dabc7a02..f74e05eb 100644 --- a/scripts/deactivate-hosted-inference.ts +++ b/scripts/deactivate-hosted-inference.ts @@ -18,17 +18,34 @@ async function main(): Promise { directDatabaseUrl: process.env.POSTIL_DIRECT_DATABASE_URL, }); delete process.env.POSTIL_DIRECT_DATABASE_URL; - const schemaReady = await getPool().query<{ ready: boolean }>( - `SELECT to_regclass('public.deployment_capabilities') IS NOT NULL - AND to_regclass('public.jobs') IS NOT NULL AS ready`, + const schema = await getPool().query<{ + hostedReady: boolean; + publicationLifecycleReady: boolean; + }>( + `SELECT + to_regclass('public.deployment_capabilities') IS NOT NULL AS "hostedReady", + to_regclass('public.deployment_capabilities') IS NOT NULL + AND to_regclass('public.jobs') IS NOT NULL + AND EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'reviews' + AND column_name = 'publication_lifecycle_required_at' + ) AS "publicationLifecycleReady"`, ); - if (schemaReady.rows[0]?.ready !== true) { + if (schema.rows[0]?.hostedReady !== true) { console.log("managed release preparation skipped until the database schema exists"); return; } - const publicationLifecycle = await deactivatePublicationLifecycleRelease( - getPool(), - ); + const publicationLifecycle = schema.rows[0].publicationLifecycleReady + ? await deactivatePublicationLifecycleRelease(getPool()) + : { deactivated: false, parked: 0 }; + const publicationLifecycleState = !schema.rows[0].publicationLifecycleReady + ? "schema not installed" + : publicationLifecycle.deactivated + ? "prior activation removed" + : "already dark"; const deactivated = await deactivateHostedInferenceRelease( getPool(), releaseSha, @@ -37,7 +54,7 @@ async function main(): Promise { `managed hosted inference prepared dark: ${deactivated ? "prior activation removed" : "already dark"}`, ); console.log( - `publication lifecycle prepared dark: ${publicationLifecycle.deactivated ? "prior activation removed" : "already dark"}; parked=${publicationLifecycle.parked}`, + `publication lifecycle prepared dark: ${publicationLifecycleState}; parked=${publicationLifecycle.parked}`, ); } finally { await closeDb(); diff --git a/tests/publication-receipt-migration.test.ts b/tests/publication-receipt-migration.test.ts index d8485858..fedd3fa1 100644 --- a/tests/publication-receipt-migration.test.ts +++ b/tests/publication-receipt-migration.test.ts @@ -719,7 +719,7 @@ describeDb("publication receipt migration and lifecycle", () => { }); }); - test("deactivation retires an idle transaction-pool session lock", async () => { + test("deactivation retires an idle transaction-pool backend and its leaked session state", async () => { const stalePool = new Pool({ connectionString: TEST_URL, max: 1, @@ -734,6 +734,10 @@ describeDb("publication receipt migration and lifecycle", () => { "SELECT pg_advisory_lock_shared(hashtextextended($1, 0))", ["postil:publication-lifecycle-release"], ); + await holder.query( + "SELECT pg_advisory_lock(hashtextextended($1, 0))", + ["postil:test-leaked-session-state"], + ); expect(await deactivatePublicationLifecycleRelease(pool)).toMatchObject({ deactivated: true, @@ -745,6 +749,17 @@ describeDb("publication receipt migration and lifecycle", () => { expect(termination?.message).toContain( "terminating connection due to administrator command", ); + const leakedState = await pool.query<{ count: string }>( + `SELECT count(*)::text AS count + FROM pg_locks + WHERE locktype = 'advisory' + AND classid::bigint = ( + (hashtextextended($1, 0) >> 32) & 4294967295 + ) + AND objid::bigint = (hashtextextended($1, 0) & 4294967295)`, + ["postil:test-leaked-session-state"], + ); + expect(leakedState.rows[0]?.count).toBe("0"); } finally { await holder .query( diff --git a/tests/release-database-url.test.ts b/tests/release-database-url.test.ts index 57cf1d95..746effa1 100644 --- a/tests/release-database-url.test.ts +++ b/tests/release-database-url.test.ts @@ -148,6 +148,7 @@ describe("release database connection", () => { expect(deployWorkflow).toContain('staged+="DATABASE_URL=${DATABASE_URL}"'); expect(deployWorkflow).not.toContain("POSTIL_DIRECT_DATABASE_URL"); expect(deactivationScript).toContain("resolveDirectDatabaseUrl"); + expect(deactivationScript).toContain("publication_lifecycle_required_at"); expect(deactivationScript.indexOf("process.env.DATABASE_URL =")).toBeLessThan( deactivationScript.indexOf("getPool().query"), ); From 6578bda274f4d8aa7b65e0deba2dc45f26502f71 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 01:01:42 +0000 Subject: [PATCH 18/34] Bind release quiescence to migrations --- package.json | 2 +- scripts/run-release-migrations.ts | 43 ++++++++++++++++++++++++------ tests/release-database-url.test.ts | 35 +++++++++++++++--------- 3 files changed, 58 insertions(+), 22 deletions(-) diff --git a/package.json b/package.json index b77cd486..77ed1ea2 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "db:push": "drizzle-kit push", "db:migrate": "drizzle-kit migrate", "db:migrate:release": "bun run scripts/run-release-migrations.ts", - "release:prepare": "bun run hosted:deactivate-release && bun run db:migrate:release && bun run operational:indexes && bun run notifications:quiesce", + "release:prepare": "bun run db:migrate:release && bun run operational:indexes && bun run notifications:quiesce", "seed": "bun run scripts/seed.ts", "billing:grant-credit": "bun run scripts/grant-billing-credit.ts", "billing:set-entitlement": "bun run scripts/set-org-entitlement.ts", diff --git a/scripts/run-release-migrations.ts b/scripts/run-release-migrations.ts index 46321259..8ebf07ca 100644 --- a/scripts/run-release-migrations.ts +++ b/scripts/run-release-migrations.ts @@ -2,7 +2,10 @@ import { resolveDirectDatabaseUrl } from "./resolve-direct-database-url"; type Environment = Record; type MigrationProcess = { exited: Promise }; -type SpawnMigration = (environment: Environment) => MigrationProcess; +type SpawnReleaseDatabaseCommand = ( + command: readonly string[], + environment: Environment, +) => MigrationProcess; export function releaseMigrationEnvironment(environment: Environment): Environment { const { POSTIL_DIRECT_DATABASE_URL: directDatabaseUrl, ...migrationEnvironment } = environment; @@ -17,28 +20,52 @@ export function releaseMigrationEnvironment(environment: Environment): Environme export async function runReleaseMigrations( environment: Environment = process.env, - spawnMigration: SpawnMigration = defaultSpawnMigration, + spawnCommand: SpawnReleaseDatabaseCommand = defaultSpawnReleaseDatabaseCommand, +): Promise { + const databaseEnvironment = releaseMigrationEnvironment(environment); + await runReleaseDatabaseCommand( + ["bun", "run", "hosted:deactivate-release"], + "release database deactivation", + databaseEnvironment, + spawnCommand, + ); + await runReleaseDatabaseCommand( + ["bun", "run", "db:migrate"], + "release database migration", + databaseEnvironment, + spawnCommand, + ); +} + +async function runReleaseDatabaseCommand( + command: readonly string[], + label: string, + environment: Environment, + spawnCommand: SpawnReleaseDatabaseCommand, ): Promise { let process: MigrationProcess; try { - process = spawnMigration(releaseMigrationEnvironment(environment)); + process = spawnCommand(command, environment); } catch (cause) { - throw new Error("release database migration could not start", { cause }); + throw new Error(`${label} could not start`, { cause }); } let exitCode: number; try { exitCode = await process.exited; } catch (cause) { - throw new Error("release database migration status could not be observed", { cause }); + throw new Error(`${label} status could not be observed`, { cause }); } if (exitCode !== 0) { - throw new Error(`release database migration failed with status ${exitCode}`); + throw new Error(`${label} failed with status ${exitCode}`); } } -function defaultSpawnMigration(environment: Environment): MigrationProcess { - return Bun.spawn(["bun", "run", "db:migrate"], { +function defaultSpawnReleaseDatabaseCommand( + command: readonly string[], + environment: Environment, +): MigrationProcess { + return Bun.spawn([...command], { env: environment, stdin: "inherit", stdout: "inherit", diff --git a/tests/release-database-url.test.ts b/tests/release-database-url.test.ts index 746effa1..1d123d36 100644 --- a/tests/release-database-url.test.ts +++ b/tests/release-database-url.test.ts @@ -55,12 +55,18 @@ describe("release database connection", () => { POSTIL_DB_POOL_MAX: "2", }; let childEnvironment: Record | undefined; + const commands: Array = []; - await runReleaseMigrations(parentEnvironment, (environment) => { + await runReleaseMigrations(parentEnvironment, (command, environment) => { + commands.push(command); childEnvironment = environment; return { exited: Promise.resolve(0) }; }); + expect(commands).toEqual([ + ["bun", "run", "hosted:deactivate-release"], + ["bun", "run", "db:migrate"], + ]); expect(parentEnvironment.DATABASE_URL).toBe(runtimeUrl); expect(childEnvironment?.DATABASE_URL).toBe(new URL(directUrl).toString()); expect(childEnvironment?.POSTIL_DIRECT_DATABASE_URL).toBeUndefined(); @@ -78,7 +84,7 @@ describe("release database connection", () => { try { await writeFile( fakeBun, - `#!${process.execPath}\nawait Bun.write(process.env.POSTIL_TEST_CAPTURE_PATH, JSON.stringify({ arguments: process.argv.slice(2), databaseUrl: process.env.DATABASE_URL, hasDirectDatabaseUrl: "POSTIL_DIRECT_DATABASE_URL" in process.env }));\n`, + `#!${process.execPath}\nconst path = process.env.POSTIL_TEST_CAPTURE_PATH; let entries = []; try { entries = JSON.parse(await Bun.file(path).text()); } catch {} entries.push({ arguments: process.argv.slice(2), databaseUrl: process.env.DATABASE_URL, hasDirectDatabaseUrl: "POSTIL_DIRECT_DATABASE_URL" in process.env }); await Bun.write(path, JSON.stringify(entries));\n`, ); await chmod(fakeBun, 0o755); @@ -100,15 +106,20 @@ describe("release database connection", () => { const stderr = await new Response(wrapper.stderr).text(); expect(exitCode, stderr).toBe(0); - const capture = JSON.parse(await readFile(capturePath, "utf8")) as { + const capture = JSON.parse(await readFile(capturePath, "utf8")) as Array<{ arguments: string[]; databaseUrl: string; hasDirectDatabaseUrl: boolean; - }; - expect(capture.arguments).toEqual(["run", "db:migrate"]); - expect(new URL(capture.databaseUrl).port).toBe("5432"); - expect(new URL(capture.databaseUrl).searchParams.has("pgbouncer")).toBe(false); - expect(capture.hasDirectDatabaseUrl).toBe(false); + }>; + expect(capture.map((entry) => entry.arguments)).toEqual([ + ["run", "hosted:deactivate-release"], + ["run", "db:migrate"], + ]); + for (const entry of capture) { + expect(new URL(entry.databaseUrl).port).toBe("5432"); + expect(new URL(entry.databaseUrl).searchParams.has("pgbouncer")).toBe(false); + expect(entry.hasDirectDatabaseUrl).toBe(false); + } } finally { await rm(temporaryDirectory, { recursive: true, force: true }); } @@ -122,10 +133,10 @@ describe("release database connection", () => { runReleaseMigrations(environment, () => { throw new Error("spawn failed"); }), - ).rejects.toThrow("release database migration could not start"); + ).rejects.toThrow("release database deactivation could not start"); await expect( runReleaseMigrations(environment, () => ({ exited: Promise.reject(new Error("lost child")) })), - ).rejects.toThrow("release database migration status could not be observed"); + ).rejects.toThrow("release database deactivation status could not be observed"); }); test("keeps the checked-in release and deploy contracts aligned", async () => { @@ -139,9 +150,7 @@ describe("release database connection", () => { "utf8", ); - expect(packageJson.scripts["release:prepare"]).toStartWith( - "bun run hosted:deactivate-release && bun run db:migrate:release", - ); + expect(packageJson.scripts["release:prepare"]).toStartWith("bun run db:migrate:release"); expect(packageJson.scripts["db:migrate:release"]).toBe( "bun run scripts/run-release-migrations.ts", ); From 0c1f114680756973b4ec9c9ddf140da50a880afc Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 01:12:46 +0000 Subject: [PATCH 19/34] Restore capabilities after failed preparation --- package.json | 2 +- scripts/run-release-migrations.ts | 116 ++++++++++++++-- src/lib/release-job-rollout.ts | 143 ++++++++++++++++++++ tests/migration-lint.test.ts | 14 +- tests/private-worker-gates.test.ts | 1 + tests/publication-receipt-migration.test.ts | 136 +++++++++++++++++++ tests/release-database-url.test.ts | 54 +++++++- 7 files changed, 447 insertions(+), 19 deletions(-) diff --git a/package.json b/package.json index 77ed1ea2..fd03dd29 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "db:push": "drizzle-kit push", "db:migrate": "drizzle-kit migrate", "db:migrate:release": "bun run scripts/run-release-migrations.ts", - "release:prepare": "bun run db:migrate:release && bun run operational:indexes && bun run notifications:quiesce", + "release:prepare": "bun run db:migrate:release", "seed": "bun run scripts/seed.ts", "billing:grant-credit": "bun run scripts/grant-billing-credit.ts", "billing:set-entitlement": "bun run scripts/set-org-entitlement.ts", diff --git a/scripts/run-release-migrations.ts b/scripts/run-release-migrations.ts index 8ebf07ca..eb5b6ec8 100644 --- a/scripts/run-release-migrations.ts +++ b/scripts/run-release-migrations.ts @@ -1,3 +1,10 @@ +import { Pool } from "pg"; + +import { + type ManagedReleaseCapabilitySnapshot, + prepareManagedReleaseCapabilities, + restoreManagedReleaseCapabilities, +} from "@/lib/release-job-rollout"; import { resolveDirectDatabaseUrl } from "./resolve-direct-database-url"; type Environment = Record; @@ -6,6 +13,13 @@ type SpawnReleaseDatabaseCommand = ( command: readonly string[], environment: Environment, ) => MigrationProcess; +type PrepareReleaseCapabilities = ( + environment: Environment, +) => Promise; +type RestoreReleaseCapabilities = ( + environment: Environment, + snapshot: ManagedReleaseCapabilitySnapshot, +) => Promise; export function releaseMigrationEnvironment(environment: Environment): Environment { const { POSTIL_DIRECT_DATABASE_URL: directDatabaseUrl, ...migrationEnvironment } = environment; @@ -21,20 +35,100 @@ export function releaseMigrationEnvironment(environment: Environment): Environme export async function runReleaseMigrations( environment: Environment = process.env, spawnCommand: SpawnReleaseDatabaseCommand = defaultSpawnReleaseDatabaseCommand, + prepareCapabilities: PrepareReleaseCapabilities = defaultPrepareReleaseCapabilities, + restoreCapabilities: RestoreReleaseCapabilities = defaultRestoreReleaseCapabilities, ): Promise { const databaseEnvironment = releaseMigrationEnvironment(environment); - await runReleaseDatabaseCommand( - ["bun", "run", "hosted:deactivate-release"], - "release database deactivation", - databaseEnvironment, - spawnCommand, - ); - await runReleaseDatabaseCommand( - ["bun", "run", "db:migrate"], - "release database migration", - databaseEnvironment, - spawnCommand, + const snapshot = await prepareCapabilities(databaseEnvironment); + try { + await runReleaseDatabaseCommand( + ["bun", "run", "db:migrate"], + "release database migration", + databaseEnvironment, + spawnCommand, + ); + await runReleaseDatabaseCommand( + ["bun", "run", "operational:indexes"], + "release operational indexes", + databaseEnvironment, + spawnCommand, + ); + await runReleaseDatabaseCommand( + ["bun", "run", "notifications:quiesce"], + "release notification quiescence", + databaseEnvironment, + spawnCommand, + ); + } catch (error) { + if (snapshot) { + try { + await restoreCapabilities(databaseEnvironment, snapshot); + } catch (restoreError) { + throw new AggregateError( + [error, restoreError], + "release database preparation and capability compensation failed", + ); + } + } + throw error; + } +} + +async function releaseSchemaState(pool: Pool): Promise<{ + hostedReady: boolean; + publicationLifecycleReady: boolean; +}> { + const result = await pool.query<{ + hostedReady: boolean; + publicationLifecycleReady: boolean; + }>( + `SELECT + to_regclass('public.deployment_capabilities') IS NOT NULL AS "hostedReady", + to_regclass('public.deployment_capabilities') IS NOT NULL + AND to_regclass('public.jobs') IS NOT NULL + AND EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'reviews' + AND column_name = 'publication_lifecycle_required_at' + ) AS "publicationLifecycleReady"`, ); + return result.rows[0] ?? { + hostedReady: false, + publicationLifecycleReady: false, + }; +} + +async function defaultPrepareReleaseCapabilities( + environment: Environment, +): Promise { + const releaseSha = environment.POSTIL_RELEASE_SHA?.trim(); + if (!releaseSha) return undefined; + const pool = new Pool({ connectionString: environment.DATABASE_URL }); + try { + const schema = await releaseSchemaState(pool); + if (!schema.hostedReady) return undefined; + return await prepareManagedReleaseCapabilities( + pool, + releaseSha, + schema.publicationLifecycleReady, + ); + } finally { + await pool.end(); + } +} + +async function defaultRestoreReleaseCapabilities( + environment: Environment, + snapshot: ManagedReleaseCapabilitySnapshot, +): Promise { + const pool = new Pool({ connectionString: environment.DATABASE_URL }); + try { + await restoreManagedReleaseCapabilities(pool, snapshot); + } finally { + await pool.end(); + } } async function runReleaseDatabaseCommand( diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index 349a13d9..2a29a205 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -42,6 +42,15 @@ async function lockPublicationLifecycleExclusive( client: PoolClient, ): Promise { const deadline = Date.now() + PUBLICATION_LIFECYCLE_LOCK_TIMEOUT_MS; + const configuredLockTimeout = await client.query<{ lock_timeout: string }>( + "SHOW lock_timeout", + ); + const lockTimeout = configuredLockTimeout.rows[0]?.lock_timeout; + if (lockTimeout === undefined) { + throw new Error( + "publication lifecycle lock timeout configuration is unavailable", + ); + } while (true) { const acquired = await client.query<{ acquired: boolean }>( "SELECT pg_try_advisory_xact_lock(hashtextextended($1, 0)) AS acquired", @@ -89,6 +98,9 @@ async function lockPublicationLifecycleExclusive( "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [PUBLICATION_LIFECYCLE_LOCK], ); + await client.query("SELECT set_config('lock_timeout', $1, true)", [ + lockTimeout, + ]); await client.query("RELEASE SAVEPOINT publication_lifecycle_lock_attempt"); return; } catch (error) { @@ -619,6 +631,137 @@ export async function deactivateHostedInferenceRelease( } } +export interface ManagedReleaseCapabilitySnapshot { + releaseSha: string; + publicationLifecycleReady: boolean; + capabilities: string[]; +} + +function managedReleaseCapabilityNames(releaseSha: string): string[] { + return [ + PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY, + HOSTED_INFERENCE_FLEET_ACTIVE_CAPABILITY, + hostedInferenceCapability(releaseSha), + hostedInferenceDarkCapability(releaseSha), + ]; +} + +/** Darken one release and retain the exact capability state for compensation. */ +export async function prepareManagedReleaseCapabilities( + pool: Pool, + releaseSha: string, + publicationLifecycleReady: boolean, +): Promise { + const normalizedRelease = normalizedReleaseSha(releaseSha); + const names = managedReleaseCapabilityNames(normalizedRelease); + const existing = await pool.query<{ name: string }>( + "SELECT name FROM deployment_capabilities WHERE name = ANY($1::text[]) ORDER BY name", + [names], + ); + const snapshot: ManagedReleaseCapabilitySnapshot = { + releaseSha: normalizedRelease, + publicationLifecycleReady, + capabilities: existing.rows.map((row) => row.name), + }; + try { + if (publicationLifecycleReady) { + await deactivatePublicationLifecycleRelease(pool); + } + await deactivateHostedInferenceRelease(pool, normalizedRelease); + return snapshot; + } catch (error) { + try { + await restoreManagedReleaseCapabilities(pool, snapshot); + } catch (restoreError) { + throw new AggregateError( + [ + databaseClientError(error, "managed release deactivation failed"), + databaseClientError( + restoreError, + "managed release capability compensation failed", + ), + ], + "managed release deactivation and capability compensation failed", + ); + } + throw error; + } +} + +/** Restore only the release capabilities changed during preparation. */ +export async function restoreManagedReleaseCapabilities( + pool: Pool, + snapshot: ManagedReleaseCapabilitySnapshot, +): Promise { + const names = managedReleaseCapabilityNames(snapshot.releaseSha); + const expected = new Set(names); + if ( + snapshot.capabilities.some((name) => !expected.has(name)) || + new Set(snapshot.capabilities).size !== snapshot.capabilities.length + ) { + throw new Error("managed release capability snapshot is invalid"); + } + const publicationWasActive = snapshot.capabilities.includes( + PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY, + ); + const client = await pool.connect(); + let releaseError: Error | undefined; + try { + await client.query("BEGIN"); + if (snapshot.publicationLifecycleReady) { + await lockPublicationLifecycleExclusive(client); + } + await client.query( + "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", + [HOSTED_INFERENCE_LOCK], + ); + await client.query( + "DELETE FROM deployment_capabilities WHERE name = ANY($1::text[])", + [names], + ); + if (snapshot.capabilities.length > 0) { + await client.query( + `INSERT INTO deployment_capabilities (name) + SELECT unnest($1::text[])`, + [snapshot.capabilities], + ); + } + if (snapshot.publicationLifecycleReady && publicationWasActive) { + await client.query( + `UPDATE jobs + SET run_after = now(), payload = payload - $1 + WHERE kind = 'gate-state-sync' + AND status = 'queued' + AND payload ? $1`, + [PUBLICATION_LIFECYCLE_DARK_PAYLOAD_KEY], + ); + } + await client.query("COMMIT"); + } catch (error) { + try { + await client.query("ROLLBACK"); + } catch (rollbackError) { + releaseError = databaseClientError( + rollbackError, + "managed release capability compensation rollback failed", + ); + throw new AggregateError( + [ + databaseClientError( + error, + "managed release capability compensation failed", + ), + releaseError, + ], + "managed release capability compensation and rollback failed", + ); + } + throw error; + } finally { + client.release(releaseError); + } +} + /** Atomically park a claimed hosted review until a verified managed release activates. */ export async function deferHostedReviewForRelease( pool: Pool, diff --git a/tests/migration-lint.test.ts b/tests/migration-lint.test.ts index a265be9d..1373c10d 100644 --- a/tests/migration-lint.test.ts +++ b/tests/migration-lint.test.ts @@ -384,6 +384,10 @@ describe("migration lint", () => { join(import.meta.dir, "..", "scripts", "ensure-operational-indexes.ts"), "utf8", ); + const releasePreparationScript = await readFile( + join(import.meta.dir, "..", "scripts", "run-release-migrations.ts"), + "utf8", + ); const packageJson = JSON.parse( await readFile(join(import.meta.dir, "..", "package.json"), "utf8"), ) as { scripts: Record }; @@ -496,8 +500,14 @@ describe("migration lint", () => { 'CREATE TABLE IF NOT EXISTS "release_steps"', ); expect(releaseScript).toContain("INSERT INTO release_steps"); - expect(packageJson.scripts["release:prepare"]).toContain( - "operational:indexes", + expect(packageJson.scripts["release:prepare"]).toBe( + "bun run db:migrate:release", + ); + expect(releasePreparationScript).toContain( + '["bun", "run", "operational:indexes"]', + ); + expect(releasePreparationScript).toContain( + '["bun", "run", "notifications:quiesce"]', ); }); diff --git a/tests/private-worker-gates.test.ts b/tests/private-worker-gates.test.ts index f0765672..da9ba2e3 100644 --- a/tests/private-worker-gates.test.ts +++ b/tests/private-worker-gates.test.ts @@ -226,6 +226,7 @@ describe("private repository worker defense in depth", () => { expect(activation).toContain("lockPublicationLifecycleExclusive(client)"); expect(exclusiveLock).toContain("pg_try_advisory_xact_lock"); expect(exclusiveLock).toContain("lock_timeout = '250ms'"); + expect(exclusiveLock).toContain("set_config('lock_timeout', $1, true)"); expect(exclusiveLock).toContain("ROLLBACK TO SAVEPOINT"); expect(exclusiveLock).toContain("pg_terminate_backend"); expect(exclusiveLock).toContain("publication lifecycle lock did not quiesce"); diff --git a/tests/publication-receipt-migration.test.ts b/tests/publication-receipt-migration.test.ts index fedd3fa1..ca5ed670 100644 --- a/tests/publication-receipt-migration.test.ts +++ b/tests/publication-receipt-migration.test.ts @@ -27,7 +27,9 @@ import { withReviewDecisionScopeLock } from "@/lib/finding-approvals"; import { activatePublicationLifecycleRelease, deactivatePublicationLifecycleRelease, + prepareManagedReleaseCapabilities, publicationLifecycleReleaseActivated, + restoreManagedReleaseCapabilities, withPublicationLifecycleReleaseActive, } from "@/lib/release-job-rollout"; @@ -719,6 +721,68 @@ describeDb("publication receipt migration and lifecycle", () => { }); }); + test("queued lifecycle acquisition restores the transaction lock timeout", async () => { + await pool.query( + `INSERT INTO deployment_capabilities (name) + VALUES ('publication-lifecycle-fleet-active') + ON CONFLICT (name) DO NOTHING`, + ); + const holderPool = new Pool({ connectionString: TEST_URL, max: 1 }); + const transitionPool = new Pool({ connectionString: TEST_URL, max: 1 }); + const rowLockPool = new Pool({ connectionString: TEST_URL, max: 1 }); + let releaseHolder!: () => void; + const holderReleased = new Promise((resolve) => { + releaseHolder = resolve; + }); + let holderAcquired!: () => void; + const acquired = new Promise((resolve) => { + holderAcquired = resolve; + }); + const holder = withPublicationLifecycleReleaseActive( + holderPool, + async () => { + holderAcquired(); + await holderReleased; + }, + ); + const rowLock = await rowLockPool.connect(); + try { + await acquired; + await rowLock.query("BEGIN"); + await rowLock.query( + `SELECT name FROM deployment_capabilities + WHERE name = 'publication-lifecycle-fleet-active' + FOR UPDATE`, + ); + const deactivation = deactivatePublicationLifecycleRelease( + transitionPool, + ).then( + (result) => ({ result, error: undefined }), + (error: unknown) => ({ result: undefined, error }), + ); + await Bun.sleep(50); + releaseHolder(); + await holder; + await Bun.sleep(400); + await rowLock.query("COMMIT"); + + const outcome = await deactivation; + expect(outcome.error).toBeUndefined(); + expect(outcome.result).toMatchObject({ deactivated: true }); + } finally { + releaseHolder(); + await holder.catch(() => undefined); + await rowLock.query("ROLLBACK").catch(() => undefined); + rowLock.release(); + await Promise.all([ + holderPool.end(), + transitionPool.end(), + rowLockPool.end(), + ]); + await activatePublicationLifecycleRelease(pool); + } + }); + test("deactivation retires an idle transaction-pool backend and its leaked session state", async () => { const stalePool = new Pool({ connectionString: TEST_URL, @@ -773,6 +837,78 @@ describeDb("publication receipt migration and lifecycle", () => { } }); + test("failed release preparation restores the exact fleet capabilities", async () => { + const releaseSha = "8".repeat(40); + const capabilityNames = [ + "publication-lifecycle-fleet-active", + "hosted-inference-fleet-active", + `hosted-inference-release:${releaseSha}`, + `hosted-inference-dark:${releaseSha}`, + ]; + await pool.query( + "DELETE FROM deployment_capabilities WHERE name = ANY($1::text[])", + [capabilityNames], + ); + await pool.query( + `INSERT INTO deployment_capabilities (name) + VALUES ('publication-lifecycle-fleet-active'), + ('hosted-inference-fleet-active'), + ($1)`, + [`hosted-inference-release:${releaseSha}`], + ); + const gate = await pool.query<{ id: string }>( + `INSERT INTO jobs (kind, payload) + VALUES ('gate-state-sync', '{"reviewId":1,"reviewPublicId":"release-compensation"}'::jsonb) + RETURNING id`, + ); + + const snapshot = await prepareManagedReleaseCapabilities( + pool, + releaseSha, + true, + ); + expect( + ( + await pool.query<{ name: string }>( + "SELECT name FROM deployment_capabilities WHERE name = ANY($1::text[]) ORDER BY name", + [capabilityNames], + ) + ).rows.map((row) => row.name), + ).toEqual([`hosted-inference-dark:${releaseSha}`]); + expect( + ( + await pool.query<{ parked: boolean }>( + "SELECT run_after = 'infinity'::timestamptz AS parked FROM jobs WHERE id = $1", + [gate.rows[0]!.id], + ) + ).rows[0]?.parked, + ).toBe(true); + + await restoreManagedReleaseCapabilities(pool, snapshot); + expect( + ( + await pool.query<{ name: string }>( + "SELECT name FROM deployment_capabilities WHERE name = ANY($1::text[]) ORDER BY name", + [capabilityNames], + ) + ).rows.map((row) => row.name), + ).toEqual([ + "hosted-inference-fleet-active", + `hosted-inference-release:${releaseSha}`, + "publication-lifecycle-fleet-active", + ]); + expect( + ( + await pool.query<{ due: boolean; dark: boolean }>( + `SELECT run_after <= now() AS due, + payload ? '_postilPublicationLifecycleDark' AS dark + FROM jobs WHERE id = $1`, + [gate.rows[0]!.id], + ) + ).rows[0], + ).toEqual({ due: true, dark: false }); + }); + test("a gate committed after the activation sweep self-heals", async () => { await deactivatePublicationLifecycleRelease(pool); const activationClient = await pool.connect(); diff --git a/tests/release-database-url.test.ts b/tests/release-database-url.test.ts index 1d123d36..a0e7d506 100644 --- a/tests/release-database-url.test.ts +++ b/tests/release-database-url.test.ts @@ -64,8 +64,9 @@ describe("release database connection", () => { }); expect(commands).toEqual([ - ["bun", "run", "hosted:deactivate-release"], ["bun", "run", "db:migrate"], + ["bun", "run", "operational:indexes"], + ["bun", "run", "notifications:quiesce"], ]); expect(parentEnvironment.DATABASE_URL).toBe(runtimeUrl); expect(childEnvironment?.DATABASE_URL).toBe(new URL(directUrl).toString()); @@ -112,8 +113,9 @@ describe("release database connection", () => { hasDirectDatabaseUrl: boolean; }>; expect(capture.map((entry) => entry.arguments)).toEqual([ - ["run", "hosted:deactivate-release"], ["run", "db:migrate"], + ["run", "operational:indexes"], + ["run", "notifications:quiesce"], ]); for (const entry of capture) { expect(new URL(entry.databaseUrl).port).toBe("5432"); @@ -133,10 +135,50 @@ describe("release database connection", () => { runReleaseMigrations(environment, () => { throw new Error("spawn failed"); }), - ).rejects.toThrow("release database deactivation could not start"); + ).rejects.toThrow("release database migration could not start"); await expect( runReleaseMigrations(environment, () => ({ exited: Promise.reject(new Error("lost child")) })), - ).rejects.toThrow("release database deactivation status could not be observed"); + ).rejects.toThrow("release database migration status could not be observed"); + }); + + test("restores the captured capability state when any database preparation step fails", async () => { + const environment = { + DATABASE_URL: "postgresql://postil@db.internal:5432/postil", + POSTIL_RELEASE_SHA: "a".repeat(40), + }; + const snapshot = { + releaseSha: "a".repeat(40), + publicationLifecycleReady: true, + capabilities: [ + "publication-lifecycle-fleet-active", + "hosted-inference-fleet-active", + ], + }; + const commands: string[][] = []; + const restored: unknown[] = []; + + await expect( + runReleaseMigrations( + environment, + (command) => { + commands.push([...command]); + return { + exited: Promise.resolve( + command.includes("operational:indexes") ? 17 : 0, + ), + }; + }, + async () => snapshot, + async (_databaseEnvironment, captured) => { + restored.push(captured); + }, + ), + ).rejects.toThrow("release operational indexes failed with status 17"); + expect(commands).toEqual([ + ["bun", "run", "db:migrate"], + ["bun", "run", "operational:indexes"], + ]); + expect(restored).toEqual([snapshot]); }); test("keeps the checked-in release and deploy contracts aligned", async () => { @@ -150,7 +192,9 @@ describe("release database connection", () => { "utf8", ); - expect(packageJson.scripts["release:prepare"]).toStartWith("bun run db:migrate:release"); + expect(packageJson.scripts["release:prepare"]).toBe( + "bun run db:migrate:release", + ); expect(packageJson.scripts["db:migrate:release"]).toBe( "bun run scripts/run-release-migrations.ts", ); From d04374c3003b801d0b49f96271dfb3db7d86e54c Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 01:40:51 +0000 Subject: [PATCH 20/34] Harden failed release preparation recovery --- .github/workflows/deploy.yml | 34 ++ .github/workflows/production-monitor.yml | 77 ++++ ...ication_lifecycle_nonblocking_triggers.sql | 71 +--- scripts/run-release-migrations.ts | 75 +++- src/lib/publication-lifecycle-lock.ts | 2 +- src/lib/release-job-rollout.ts | 385 ++++++++++++++---- src/worker/index.ts | 11 + tests/migration-lint.test.ts | 42 +- tests/private-worker-gates.test.ts | 5 +- tests/publication-receipt-migration.test.ts | 161 +++++++- tests/release-database-url.test.ts | 40 ++ tests/worker-runner.test.ts | 2 + 12 files changed, 712 insertions(+), 193 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 1ff6f98e..f113cef2 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -326,6 +326,40 @@ jobs: fi env: FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + - name: Restore capabilities when release preparation failed before replacement + if: ${{ always() && steps.deploy.outcome == 'failure' && steps.recover.outcome == 'success' }} + timeout-minutes: 5 + run: | + set -euo pipefail + machines=$(flyctl machine list --app postil-web --json) + target_seen=0 + while IFS= read -r id; do + release=$(flyctl machine exec "${id}" \ + "bun -e 'process.stdout.write(process.env.POSTIL_RELEASE_SHA ?? \"\")'" \ + --app postil-web --timeout 15 2>/dev/null || true) + if [[ ! "${release}" =~ ^[0-9a-f]{7,40}$ ]]; then + echo "A managed machine did not report a valid release; capabilities remain dark." + exit 1 + fi + if [[ "${release}" == "${GITHUB_SHA}" ]]; then + target_seen=1 + break + fi + done < <(jq -r '.[] | select( + .state == "started" and + (.config.metadata.fly_process_group == "web" or + .config.metadata.fly_process_group == "worker" or + .config.metadata.fly_process_group == "monitor") + ) | .id' <<<"${machines}") + if [[ "${target_seen}" -ne 0 ]]; then + echo "A target-release machine is running; release capabilities remain dark." + exit 0 + fi + bun scripts/run-release-migrations.ts --compensate + env: + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTIL_RELEASE_SHA: ${{ github.sha }} - name: Verify and activate release capabilities after fleet replacement id: activate # Migration 0020 stages new job kinds with an infinite diff --git a/.github/workflows/production-monitor.yml b/.github/workflows/production-monitor.yml index b50c43c5..c8e578b4 100644 --- a/.github/workflows/production-monitor.yml +++ b/.github/workflows/production-monitor.yml @@ -1,6 +1,9 @@ name: Production monitor on: + workflow_run: + workflows: ["deploy"] + types: [completed] schedule: # Requested every 15 minutes; GitHub throttles scheduled workflows, so # observed cadence is best-effort (often hourly or worse). This workflow @@ -24,8 +27,82 @@ concurrency: cancel-in-progress: false jobs: + release-recovery: + name: Recover abandoned release preparation + if: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.conclusion != 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 6 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.3.14 + - name: Install dependencies + run: bun install --frozen-lockfile + - name: Install checksum-pinned flyctl + env: + FLYCTL_VERSION: 0.4.71 + FLYCTL_LINUX_X86_64_SHA256: a782dceed173d215c000ab94e2b08623c22267edff6d90ebe3010b3f9b671dc2 + run: | + set -euo pipefail + archive="flyctl_${FLYCTL_VERSION}_Linux_x86_64.tar.gz" + url="https://github.com/superfly/flyctl/releases/download/v${FLYCTL_VERSION}/${archive}" + temporary_directory="$(mktemp -d)" + trap 'rm -rf "${temporary_directory}"' EXIT + curl --fail --location --silent --show-error \ + --retry 5 --retry-all-errors --retry-delay 2 \ + --output "${temporary_directory}/${archive}" "${url}" + printf '%s %s\n' "${FLYCTL_LINUX_X86_64_SHA256}" "${temporary_directory}/${archive}" \ + | sha256sum --check --strict + tar -xzf "${temporary_directory}/${archive}" -C "${temporary_directory}" flyctl + install -m 0755 "${temporary_directory}/flyctl" "${RUNNER_TEMP}/flyctl" + echo "${RUNNER_TEMP}" >> "${GITHUB_PATH}" + - name: Restore only an unchanged prior fleet + env: + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTIL_RELEASE_SHA: ${{ github.event.workflow_run.head_sha }} + run: | + set -euo pipefail + machines=$(flyctl machine list --app postil-web --json) + managed_count=$(jq -r '[.[] | select( + .config.metadata.fly_process_group == "web" or + .config.metadata.fly_process_group == "worker" or + .config.metadata.fly_process_group == "monitor" + )] | length' <<<"${machines}") + started_count=$(jq -r '[.[] | select( + .state == "started" and + (.config.metadata.fly_process_group == "web" or + .config.metadata.fly_process_group == "worker" or + .config.metadata.fly_process_group == "monitor") + )] | length' <<<"${machines}") + if [[ "${managed_count}" -lt 4 || "${started_count}" -ne "${managed_count}" ]]; then + echo "Managed fleet state is incomplete; release capabilities remain dark." + exit 1 + fi + while IFS= read -r id; do + release=$(flyctl machine exec "${id}" \ + "bun -e 'process.stdout.write(process.env.POSTIL_RELEASE_SHA ?? \"\")'" \ + --app postil-web --timeout 15 2>/dev/null) + if [[ ! "${release}" =~ ^[0-9a-f]{7,40}$ ]]; then + echo "A managed machine did not report a valid release; capabilities remain dark." + exit 1 + fi + if [[ "${release}" == "${POSTIL_RELEASE_SHA}" ]]; then + echo "The target release reached the managed fleet; capabilities remain dark." + exit 1 + fi + done < <(jq -r '.[] | select( + .state == "started" and + (.config.metadata.fly_process_group == "web" or + .config.metadata.fly_process_group == "worker" or + .config.metadata.fly_process_group == "monitor") + ) | .id' <<<"${machines}") + bun scripts/run-release-migrations.ts --compensate + smoke: name: Smoke check production + if: ${{ github.event_name != 'workflow_run' }} runs-on: ubuntu-latest timeout-minutes: 6 steps: diff --git a/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql b/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql index 087e9b76..82369d67 100644 --- a/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql +++ b/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql @@ -1,61 +1,3 @@ --- Transaction-pool workers can leave session state attached to an idle --- Supavisor backend after the logical client returns to the pool. A backend --- selected by the exact lifecycle lock has no active query, so terminating it --- intentionally discards all leaked session state on that backend. Active --- sessions and backends without the exact lifecycle lock remain untouched. -DO $$ -DECLARE - lifecycle_locked boolean := false; - stale_pid integer; - stale_state text; - cleanup_deadline timestamptz := clock_timestamp() + interval '30 seconds'; -BEGIN - LOOP - lifecycle_locked := pg_try_advisory_lock( - hashtextextended('postil:publication-lifecycle-release', 0) - ); - EXIT WHEN lifecycle_locked; - - PERFORM pg_stat_clear_snapshot(); - stale_pid := NULL; - stale_state := NULL; - SELECT advisory.pid, activity.state - INTO stale_pid, stale_state - FROM pg_locks AS advisory - INNER JOIN pg_stat_activity AS activity ON activity.pid = advisory.pid - WHERE advisory.locktype = 'advisory' - AND advisory.granted - AND advisory.mode IN ('ShareLock', 'ExclusiveLock') - AND advisory.objsubid = 1 - AND activity.datname = current_database() - AND activity.usename = current_user - AND advisory.classid::bigint = ( - (hashtextextended('postil:publication-lifecycle-release', 0) >> 32) - & 4294967295 - ) - AND advisory.objid::bigint = ( - hashtextextended('postil:publication-lifecycle-release', 0) - & 4294967295 - ) - AND advisory.pid <> pg_backend_pid() - AND activity.application_name = 'Supavisor' - ORDER BY (activity.state = 'idle') DESC, advisory.pid - LIMIT 1; - - EXIT WHEN stale_pid IS NULL; - - IF clock_timestamp() >= cleanup_deadline THEN - RAISE EXCEPTION 'active legacy publication lifecycle lock did not quiesce'; - END IF; - IF stale_state = 'idle' THEN - PERFORM pg_terminate_backend(stale_pid); - PERFORM pg_sleep(0.05); - ELSE - PERFORM pg_sleep(0.1); - END IF; - END LOOP; -END; -$$;--> statement-breakpoint CREATE OR REPLACE FUNCTION "postil_require_publication_lifecycle"() RETURNS trigger LANGUAGE plpgsql @@ -66,7 +8,7 @@ BEGIN -- review row. The lifecycle marker is monotonic and its gate is staged by -- the companion trigger below. PERFORM pg_try_advisory_xact_lock_shared( - hashtextextended('postil:publication-lifecycle-release', 0) + hashtextextended('postil:publication-lifecycle-release-v2', 0) ); IF NEW.publication_lifecycle_required_at IS NULL AND NEW.envelope IS NOT NULL @@ -97,7 +39,7 @@ BEGIN -- A failed try-lock means deactivation owns or is queued for the boundary. -- Park the job without waiting while its caller may hold narrower locks. lifecycle_locked := pg_try_advisory_xact_lock_shared( - hashtextextended('postil:publication-lifecycle-release', 0) + hashtextextended('postil:publication-lifecycle-release-v2', 0) ); IF lifecycle_locked THEN SELECT EXISTS ( @@ -122,13 +64,4 @@ BEGIN END IF; RETURN NEW; END; -$$;--> statement-breakpoint -DO $$ -BEGIN - IF NOT pg_advisory_unlock( - hashtextextended('postil:publication-lifecycle-release', 0) - ) THEN - RAISE EXCEPTION 'publication lifecycle migration lock was not held'; - END IF; -END; $$; diff --git a/scripts/run-release-migrations.ts b/scripts/run-release-migrations.ts index eb5b6ec8..3c3e18fb 100644 --- a/scripts/run-release-migrations.ts +++ b/scripts/run-release-migrations.ts @@ -3,12 +3,16 @@ import { Pool } from "pg"; import { type ManagedReleaseCapabilitySnapshot, prepareManagedReleaseCapabilities, + restoreManagedReleasePreparation, restoreManagedReleaseCapabilities, } from "@/lib/release-job-rollout"; import { resolveDirectDatabaseUrl } from "./resolve-direct-database-url"; type Environment = Record; -type MigrationProcess = { exited: Promise }; +type MigrationProcess = { + exited: Promise; + kill?: (signal?: number | NodeJS.Signals) => unknown; +}; type SpawnReleaseDatabaseCommand = ( command: readonly string[], environment: Environment, @@ -37,6 +41,7 @@ export async function runReleaseMigrations( spawnCommand: SpawnReleaseDatabaseCommand = defaultSpawnReleaseDatabaseCommand, prepareCapabilities: PrepareReleaseCapabilities = defaultPrepareReleaseCapabilities, restoreCapabilities: RestoreReleaseCapabilities = defaultRestoreReleaseCapabilities, + signal?: AbortSignal, ): Promise { const databaseEnvironment = releaseMigrationEnvironment(environment); const snapshot = await prepareCapabilities(databaseEnvironment); @@ -46,18 +51,21 @@ export async function runReleaseMigrations( "release database migration", databaseEnvironment, spawnCommand, + signal, ); await runReleaseDatabaseCommand( ["bun", "run", "operational:indexes"], "release operational indexes", databaseEnvironment, spawnCommand, + signal, ); await runReleaseDatabaseCommand( ["bun", "run", "notifications:quiesce"], "release notification quiescence", databaseEnvironment, spawnCommand, + signal, ); } catch (error) { if (snapshot) { @@ -131,11 +139,30 @@ async function defaultRestoreReleaseCapabilities( } } +export async function compensateReleasePreparation( + environment: Environment = process.env, +): Promise { + const releaseSha = environment.POSTIL_RELEASE_SHA?.trim(); + if (!releaseSha) { + throw new Error("POSTIL_RELEASE_SHA is required for release compensation"); + } + const databaseEnvironment = releaseMigrationEnvironment(environment); + const pool = new Pool({ connectionString: databaseEnvironment.DATABASE_URL }); + try { + const schema = await releaseSchemaState(pool); + if (!schema.hostedReady) return false; + return await restoreManagedReleasePreparation(pool, releaseSha); + } finally { + await pool.end(); + } +} + async function runReleaseDatabaseCommand( command: readonly string[], label: string, environment: Environment, spawnCommand: SpawnReleaseDatabaseCommand, + signal?: AbortSignal, ): Promise { let process: MigrationProcess; try { @@ -145,10 +172,24 @@ async function runReleaseDatabaseCommand( } let exitCode: number; + let abortHandler: (() => void) | undefined; try { - exitCode = await process.exited; + const interrupted = new Promise((_resolve, reject) => { + abortHandler = () => { + process.kill?.("SIGTERM"); + reject(new Error(`${label} interrupted`)); + }; + if (signal?.aborted) abortHandler(); + else signal?.addEventListener("abort", abortHandler, { once: true }); + }); + exitCode = await Promise.race([process.exited, interrupted]); } catch (cause) { + if (cause instanceof Error && cause.message === `${label} interrupted`) { + throw cause; + } throw new Error(`${label} status could not be observed`, { cause }); + } finally { + if (abortHandler) signal?.removeEventListener("abort", abortHandler); } if (exitCode !== 0) { throw new Error(`${label} failed with status ${exitCode}`); @@ -167,4 +208,32 @@ function defaultSpawnReleaseDatabaseCommand( }); } -if (import.meta.main) await runReleaseMigrations(); +if (import.meta.main) { + if (process.argv[2] === "--compensate") { + const restored = await compensateReleasePreparation(); + console.log( + `release preparation compensation: ${restored ? "restored" : "not pending"}`, + ); + process.exit(0); + } + const controller = new AbortController(); + const interrupt = (signal: NodeJS.Signals) => { + controller.abort(new Error(`release database preparation received ${signal}`)); + }; + const onInterrupt = () => interrupt("SIGINT"); + const onTerminate = () => interrupt("SIGTERM"); + process.once("SIGINT", onInterrupt); + process.once("SIGTERM", onTerminate); + try { + await runReleaseMigrations( + process.env, + defaultSpawnReleaseDatabaseCommand, + defaultPrepareReleaseCapabilities, + defaultRestoreReleaseCapabilities, + controller.signal, + ); + } finally { + process.off("SIGINT", onInterrupt); + process.off("SIGTERM", onTerminate); + } +} diff --git a/src/lib/publication-lifecycle-lock.ts b/src/lib/publication-lifecycle-lock.ts index 37b167b8..59a39e46 100644 --- a/src/lib/publication-lifecycle-lock.ts +++ b/src/lib/publication-lifecycle-lock.ts @@ -3,7 +3,7 @@ import { sql } from "drizzle-orm"; import type { Database } from "@/lib/db"; export const PUBLICATION_LIFECYCLE_LOCK = - "postil:publication-lifecycle-release"; + "postil:publication-lifecycle-release-v2"; /** Keep lifecycle work ahead of narrower review locks in the global order. */ export async function lockPublicationLifecycleShared( diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index 2a29a205..18520922 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -25,6 +25,8 @@ export const PRIVATE_REVIEW_AUTHOR_CAPABILITY = "private-review-author-v1"; const PRIVATE_REVIEW_AUTHOR_LOCK = "postil:private-review-author-v1"; const HOSTED_INFERENCE_CAPABILITY_PREFIX = "hosted-inference-release:"; const HOSTED_INFERENCE_DARK_PREFIX = "hosted-inference-dark:"; +const MANAGED_RELEASE_PREPARATION_PREFIX = "managed-release-preparation:"; +const MANAGED_RELEASE_RECOVERY_AGE_MS = 20 * 60 * 1000; export const HOSTED_INFERENCE_FLEET_ACTIVE_CAPABILITY = "hosted-inference-fleet-active"; export const HOSTED_INFERENCE_LOCK = "postil:hosted-inference-release"; @@ -33,6 +35,8 @@ export const PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY = const PUBLICATION_LIFECYCLE_DARK_PAYLOAD_KEY = "_postilPublicationLifecycleDark"; const PUBLICATION_LIFECYCLE_LOCK_TIMEOUT_MS = 30_000; +const LEGACY_PUBLICATION_LIFECYCLE_LOCK = + "postil:publication-lifecycle-release"; function databaseClientError(error: unknown, fallback: string): Error { return error instanceof Error ? error : new Error(fallback); @@ -58,37 +62,6 @@ async function lockPublicationLifecycleExclusive( ); if (acquired.rows[0]?.acquired === true) return; - await client.query("SELECT pg_stat_clear_snapshot()"); - const stale = await client.query<{ pid: number }>( - `SELECT advisory.pid - FROM pg_locks AS advisory - INNER JOIN pg_stat_activity AS activity ON activity.pid = advisory.pid - WHERE advisory.locktype = 'advisory' - AND advisory.granted - AND advisory.mode IN ('ShareLock', 'ExclusiveLock') - AND advisory.objsubid = 1 - AND activity.datname = current_database() - AND activity.usename = current_user - AND advisory.classid::bigint = ( - (hashtextextended($1, 0) >> 32) & 4294967295 - ) - AND advisory.objid::bigint = ( - hashtextextended($1, 0) & 4294967295 - ) - AND advisory.pid <> pg_backend_pid() - AND activity.application_name = 'Supavisor' - AND activity.state = 'idle' - ORDER BY advisory.pid - LIMIT 1`, - [PUBLICATION_LIFECYCLE_LOCK], - ); - if (stale.rows[0]) { - await client.query("SELECT pg_terminate_backend($1)", [stale.rows[0].pid]); - } - if (Date.now() >= deadline) { - throw new Error("publication lifecycle lock did not quiesce within 30 seconds"); - } - await client.query("SAVEPOINT publication_lifecycle_lock_attempt"); try { // A bounded blocking request enters PostgreSQL's lock queue. Trigger @@ -120,7 +93,54 @@ async function lockPublicationLifecycleExclusive( ); } if ((error as { code?: string }).code !== "55P03") throw error; + if (Date.now() >= deadline) { + throw new Error( + "publication lifecycle lock did not quiesce within 30 seconds", + ); + } + } + } +} + +async function waitForLegacyPublicationLifecycleTransactions( + client: PoolClient, +): Promise { + const deadline = Date.now() + PUBLICATION_LIFECYCLE_LOCK_TIMEOUT_MS; + while (true) { + await client.query("SELECT pg_stat_clear_snapshot()"); + const active = await client.query<{ active: boolean }>( + `SELECT EXISTS ( + SELECT 1 + FROM pg_locks AS advisory + INNER JOIN pg_stat_activity AS activity ON activity.pid = advisory.pid + WHERE advisory.locktype = 'advisory' + AND advisory.granted + AND advisory.mode IN ('ShareLock', 'ExclusiveLock') + AND advisory.objsubid = 1 + AND activity.datname = current_database() + AND activity.usename = current_user + AND advisory.classid::bigint = ( + (hashtextextended($1, 0) >> 32) & 4294967295 + ) + AND advisory.objid::bigint = ( + hashtextextended($1, 0) & 4294967295 + ) + AND advisory.pid <> pg_backend_pid() + AND ( + activity.state IS DISTINCT FROM 'idle' + OR activity.backend_xid IS NOT NULL + OR activity.backend_xmin IS NOT NULL + ) + ) AS active`, + [LEGACY_PUBLICATION_LIFECYCLE_LOCK], + ); + if (active.rows[0]?.active !== true) return; + if (Date.now() >= deadline) { + throw new Error( + "legacy publication lifecycle transactions did not quiesce within 30 seconds", + ); } + await client.query("SELECT pg_sleep(0.1)"); } } @@ -177,43 +197,48 @@ export async function deactivatePublicationLifecycleRelease( ): Promise<{ deactivated: boolean; parked: number }> { const client = await pool.connect(); let releaseError: Error | undefined; + let transactionOpen = false; try { + // Darken first without waiting for the legacy session lock. A pooled + // backend reassigned during cleanup then fails the capability check before + // it can publish. The exclusive pass below drains work already in flight. await client.query("BEGIN"); + transactionOpen = true; + const initial = await darkenPublicationLifecycle(client); + await client.query("COMMIT"); + transactionOpen = false; + + await client.query("BEGIN"); + transactionOpen = true; await lockPublicationLifecycleExclusive(client); - const deactivated = await client.query( - "DELETE FROM deployment_capabilities WHERE name = $1", - [PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY], - ); - const parked = await client.query( - `UPDATE jobs - SET run_after = 'infinity'::timestamptz, - payload = jsonb_set( - payload, - ARRAY[$1]::text[], - 'true'::jsonb, - true - ) - WHERE kind = 'gate-state-sync' - AND status = 'queued'`, - [PUBLICATION_LIFECYCLE_DARK_PAYLOAD_KEY], - ); + await waitForLegacyPublicationLifecycleTransactions(client); + const fenced = await darkenPublicationLifecycle(client); await client.query("COMMIT"); + transactionOpen = false; return { - deactivated: (deactivated.rowCount ?? 0) > 0, - parked: parked.rowCount ?? 0, + deactivated: initial.deactivated || fenced.deactivated, + parked: initial.parked + fenced.parked, }; } catch (error) { - try { - await client.query("ROLLBACK"); - } catch (rollbackError) { - releaseError = databaseClientError( - rollbackError, - "publication lifecycle deactivation rollback failed", - ); - throw new AggregateError( - [databaseClientError(error, "publication lifecycle deactivation failed"), releaseError], - "publication lifecycle deactivation and rollback failed", - ); + if (transactionOpen) { + try { + await client.query("ROLLBACK"); + } catch (rollbackError) { + releaseError = databaseClientError( + rollbackError, + "publication lifecycle deactivation rollback failed", + ); + throw new AggregateError( + [ + databaseClientError( + error, + "publication lifecycle deactivation failed", + ), + releaseError, + ], + "publication lifecycle deactivation and rollback failed", + ); + } } throw error; } finally { @@ -221,6 +246,36 @@ export async function deactivatePublicationLifecycleRelease( } } +async function darkenPublicationLifecycle( + client: PoolClient, +): Promise<{ deactivated: boolean; parked: number }> { + const deactivated = await client.query( + "DELETE FROM deployment_capabilities WHERE name = $1", + [PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY], + ); + const parked = await client.query( + `UPDATE jobs + SET run_after = 'infinity'::timestamptz, + payload = jsonb_set( + payload, + ARRAY[$1]::text[], + 'true'::jsonb, + true + ) + WHERE kind = 'gate-state-sync' + AND status = 'queued' + AND ( + run_after <> 'infinity'::timestamptz + OR NOT (payload ? $1) + )`, + [PUBLICATION_LIFECYCLE_DARK_PAYLOAD_KEY], + ); + return { + deactivated: (deactivated.rowCount ?? 0) > 0, + parked: parked.rowCount ?? 0, + }; +} + /** Queue mixed-fleet recovery and release gates after homogeneous-fleet proof. */ export async function activatePublicationLifecycleRelease( pool: Pool, @@ -584,6 +639,10 @@ export async function activateHostedInferenceRelease( "DELETE FROM deployment_capabilities WHERE name LIKE $1", [`${HOSTED_INFERENCE_DARK_PREFIX}%`], ); + await client.query( + "DELETE FROM deployment_capabilities WHERE name = ANY($1::text[])", + [Object.values(managedReleasePreparationNames(releaseSha))], + ); await client.query("COMMIT"); return (activated.rowCount ?? 0) > 0; } catch (error) { @@ -646,6 +705,110 @@ function managedReleaseCapabilityNames(releaseSha: string): string[] { ]; } +function managedReleasePreparationNames(releaseSha: string): { + root: string; + publicationReady: string; + publicationActive: string; + hostedFleetActive: string; + hostedReleaseActive: string; + hostedDarkActive: string; +} { + const prefix = `${MANAGED_RELEASE_PREPARATION_PREFIX}${releaseSha}:`; + return { + root: `${prefix}root`, + publicationReady: `${prefix}publication-ready`, + publicationActive: `${prefix}publication-active`, + hostedFleetActive: `${prefix}hosted-fleet-active`, + hostedReleaseActive: `${prefix}hosted-release-active`, + hostedDarkActive: `${prefix}hosted-dark-active`, + }; +} + +function managedReleasePreparationSnapshot( + releaseSha: string, + names: readonly string[], +): ManagedReleaseCapabilitySnapshot | undefined { + const journal = managedReleasePreparationNames(releaseSha); + const present = new Set(names); + if (!present.has(journal.root)) return undefined; + const capabilities: string[] = []; + if (present.has(journal.publicationActive)) { + capabilities.push(PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY); + } + if (present.has(journal.hostedFleetActive)) { + capabilities.push(HOSTED_INFERENCE_FLEET_ACTIVE_CAPABILITY); + } + if (present.has(journal.hostedReleaseActive)) { + capabilities.push(hostedInferenceCapability(releaseSha)); + } + if (present.has(journal.hostedDarkActive)) { + capabilities.push(hostedInferenceDarkCapability(releaseSha)); + } + return { + releaseSha, + publicationLifecycleReady: present.has(journal.publicationReady), + capabilities, + }; +} + +async function captureManagedReleaseCapabilities( + pool: Pool, + releaseSha: string, + publicationLifecycleReady: boolean, +): Promise { + const names = managedReleaseCapabilityNames(releaseSha); + const journal = managedReleasePreparationNames(releaseSha); + const client = await pool.connect(); + try { + await client.query("BEGIN"); + if (publicationLifecycleReady) { + await lockPublicationLifecycleExclusive(client); + } + await client.query( + "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", + [HOSTED_INFERENCE_LOCK], + ); + const existing = await client.query<{ name: string }>( + "SELECT name FROM deployment_capabilities WHERE name = ANY($1::text[]) ORDER BY name", + [names], + ); + const snapshot: ManagedReleaseCapabilitySnapshot = { + releaseSha, + publicationLifecycleReady, + capabilities: existing.rows.map((row) => row.name), + }; + const journalNames = [ + journal.root, + ...(publicationLifecycleReady ? [journal.publicationReady] : []), + ...(snapshot.capabilities.includes(PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY) + ? [journal.publicationActive] + : []), + ...(snapshot.capabilities.includes(HOSTED_INFERENCE_FLEET_ACTIVE_CAPABILITY) + ? [journal.hostedFleetActive] + : []), + ...(snapshot.capabilities.includes(hostedInferenceCapability(releaseSha)) + ? [journal.hostedReleaseActive] + : []), + ...(snapshot.capabilities.includes(hostedInferenceDarkCapability(releaseSha)) + ? [journal.hostedDarkActive] + : []), + ]; + await client.query( + `INSERT INTO deployment_capabilities (name) + SELECT unnest($1::text[]) + ON CONFLICT (name) DO UPDATE SET activated_at = now()`, + [journalNames], + ); + await client.query("COMMIT"); + return snapshot; + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw error; + } finally { + client.release(); + } +} + /** Darken one release and retain the exact capability state for compensation. */ export async function prepareManagedReleaseCapabilities( pool: Pool, @@ -653,16 +816,12 @@ export async function prepareManagedReleaseCapabilities( publicationLifecycleReady: boolean, ): Promise { const normalizedRelease = normalizedReleaseSha(releaseSha); - const names = managedReleaseCapabilityNames(normalizedRelease); - const existing = await pool.query<{ name: string }>( - "SELECT name FROM deployment_capabilities WHERE name = ANY($1::text[]) ORDER BY name", - [names], - ); - const snapshot: ManagedReleaseCapabilitySnapshot = { - releaseSha: normalizedRelease, + await restoreManagedReleasePreparation(pool, normalizedRelease); + const snapshot = await captureManagedReleaseCapabilities( + pool, + normalizedRelease, publicationLifecycleReady, - capabilities: existing.rows.map((row) => row.name), - }; + ); try { if (publicationLifecycleReady) { await deactivatePublicationLifecycleRelease(pool); @@ -693,7 +852,17 @@ export async function restoreManagedReleaseCapabilities( pool: Pool, snapshot: ManagedReleaseCapabilitySnapshot, ): Promise { + await restoreManagedReleaseCapabilitiesInternal(pool, snapshot, false); +} + +async function restoreManagedReleaseCapabilitiesInternal( + pool: Pool, + snapshot: ManagedReleaseCapabilitySnapshot, + requireJournal: boolean, +): Promise { const names = managedReleaseCapabilityNames(snapshot.releaseSha); + const journal = managedReleasePreparationNames(snapshot.releaseSha); + const journalNames = Object.values(journal); const expected = new Set(names); if ( snapshot.capabilities.some((name) => !expected.has(name)) || @@ -715,6 +884,18 @@ export async function restoreManagedReleaseCapabilities( "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [HOSTED_INFERENCE_LOCK], ); + if (requireJournal) { + const durable = await client.query<{ present: boolean }>( + `SELECT EXISTS ( + SELECT 1 FROM deployment_capabilities WHERE name = $1 + ) AS present`, + [journal.root], + ); + if (durable.rows[0]?.present !== true) { + await client.query("COMMIT"); + return false; + } + } await client.query( "DELETE FROM deployment_capabilities WHERE name = ANY($1::text[])", [names], @@ -736,7 +917,23 @@ export async function restoreManagedReleaseCapabilities( [PUBLICATION_LIFECYCLE_DARK_PAYLOAD_KEY], ); } + if (snapshot.capabilities.includes(HOSTED_INFERENCE_FLEET_ACTIVE_CAPABILITY)) { + await client.query( + `UPDATE jobs + SET run_after = now(), payload = payload - 'releaseDarkSha' + WHERE kind IN ('review', $1) + AND status = 'queued' + AND run_after = 'infinity'::timestamptz + AND payload->>'releaseDarkSha' = $2`, + [HOSTED_PROVIDER_KEY_LIFECYCLE_JOB_KIND, snapshot.releaseSha], + ); + } + await client.query( + "DELETE FROM deployment_capabilities WHERE name = ANY($1::text[])", + [journalNames], + ); await client.query("COMMIT"); + return true; } catch (error) { try { await client.query("ROLLBACK"); @@ -762,6 +959,54 @@ export async function restoreManagedReleaseCapabilities( } } +export async function restoreManagedReleasePreparation( + pool: Pool, + releaseSha: string, +): Promise { + const normalizedRelease = normalizedReleaseSha(releaseSha); + const journal = managedReleasePreparationNames(normalizedRelease); + const durable = await pool.query<{ name: string }>( + "SELECT name FROM deployment_capabilities WHERE name = ANY($1::text[])", + [Object.values(journal)], + ); + const snapshot = managedReleasePreparationSnapshot( + normalizedRelease, + durable.rows.map((row) => row.name), + ); + if (!snapshot) return false; + return restoreManagedReleaseCapabilitiesInternal(pool, snapshot, true); +} + +/** Restore release preparation abandoned by a dead deploy process. */ +export async function recoverAbandonedManagedReleasePreparations( + pool: Pool, + currentReleaseSha: string | undefined, + minimumAgeMs = MANAGED_RELEASE_RECOVERY_AGE_MS, +): Promise { + const current = currentReleaseSha + ? normalizedReleaseSha(currentReleaseSha) + : undefined; + const roots = await pool.query<{ name: string }>( + `SELECT name + FROM deployment_capabilities + WHERE name LIKE $1 + AND name LIKE '%:root' + AND activated_at <= now() - ($2::double precision * interval '1 millisecond') + ORDER BY activated_at`, + [`${MANAGED_RELEASE_PREPARATION_PREFIX}%`, minimumAgeMs], + ); + let recovered = 0; + for (const row of roots.rows) { + const releaseSha = row.name.slice( + MANAGED_RELEASE_PREPARATION_PREFIX.length, + -":root".length, + ); + if (!/^[0-9a-f]{7,40}$/.test(releaseSha) || releaseSha === current) continue; + if (await restoreManagedReleasePreparation(pool, releaseSha)) recovered += 1; + } + return recovered; +} + /** Atomically park a claimed hosted review until a verified managed release activates. */ export async function deferHostedReviewForRelease( pool: Pool, diff --git a/src/worker/index.ts b/src/worker/index.ts index 64a5a539..27cfd9bb 100644 --- a/src/worker/index.ts +++ b/src/worker/index.ts @@ -21,6 +21,7 @@ import { } from "@/lib/queue"; import { redactSecrets } from "@/lib/redact"; import { recoverRespondDeliveryJobs } from "@/lib/respond-delivery"; +import { recoverAbandonedManagedReleasePreparations } from "@/lib/release-job-rollout"; import { recordServiceHeartbeat } from "@/lib/private-monitoring"; import { configuredPrivateWorkerRehearsalSandbox, @@ -256,6 +257,16 @@ function jitter(delayMs: number): number { async function watchdogLoop(): Promise { while (!shuttingDown) { try { + const recoveredReleasePreparations = + await recoverAbandonedManagedReleasePreparations( + getPool(), + optionalEnv("POSTIL_RELEASE_SHA"), + ); + if (recoveredReleasePreparations > 0) { + console.warn( + `[watchdog] recovered ${recoveredReleasePreparations} abandoned release preparation(s)`, + ); + } await enqueueGateEnforcementSweepOnce(getPool(), { minIntervalMs: GATE_ENFORCEMENT_SWEEP_INTERVAL_MS, }); diff --git a/tests/migration-lint.test.ts b/tests/migration-lint.test.ts index 1373c10d..0114cad5 100644 --- a/tests/migration-lint.test.ts +++ b/tests/migration-lint.test.ts @@ -426,47 +426,17 @@ describe("migration lint", () => { expect(publicationLifecycleMigration).toContain( 'UPDATE "jobs"\nSET "run_after" = \'infinity\'::timestamptz', ); - expect(publicationLifecycleRepairMigration).toContain( - "PERFORM pg_terminate_backend(stale_pid)", - ); - expect(publicationLifecycleRepairMigration).toContain( - "activity.application_name = 'Supavisor'", - ); - expect(publicationLifecycleRepairMigration).toContain( - "ORDER BY (activity.state = 'idle') DESC", - ); - expect(publicationLifecycleRepairMigration).toContain( - "IF stale_state = 'idle' THEN", - ); - expect(publicationLifecycleRepairMigration).toContain( - "activity.datname = current_database()", - ); - expect(publicationLifecycleRepairMigration).toContain( - "activity.usename = current_user", - ); - expect(publicationLifecycleRepairMigration).toContain( - "clock_timestamp() + interval '30 seconds'", - ); - expect(publicationLifecycleRepairMigration).toContain( - "lifecycle_locked := pg_try_advisory_lock(", - ); - expect(publicationLifecycleRepairMigration).toContain( - "IF NOT pg_advisory_unlock(", - ); expect(publicationLifecycleRepairMigration).not.toContain( - "SELECT pg_advisory_xact_lock(hashtextextended('postil:publication-lifecycle-release'", + "pg_terminate_backend", ); - expect(publicationLifecycleRepairMigration).toContain( - "PERFORM pg_stat_clear_snapshot()", - ); - expect(publicationLifecycleRepairMigration).toContain( - "active legacy publication lifecycle lock did not quiesce", + expect(publicationLifecycleRepairMigration).not.toContain( + "pg_advisory_lock(", ); - expect(publicationLifecycleRepairMigration).toContain( - "advisory.pid <> pg_backend_pid()", + expect(publicationLifecycleRepairMigration).not.toContain( + "pg_advisory_unlock(", ); expect(publicationLifecycleRepairMigration).toContain( - "hashtextextended('postil:publication-lifecycle-release', 0)", + "hashtextextended('postil:publication-lifecycle-release-v2', 0)", ); expect(releaseScript).toContain( 'CREATE INDEX CONCURRENTLY IF NOT EXISTS "reviews_publication_lifecycle_pending_idx"', diff --git a/tests/private-worker-gates.test.ts b/tests/private-worker-gates.test.ts index da9ba2e3..55a7cc14 100644 --- a/tests/private-worker-gates.test.ts +++ b/tests/private-worker-gates.test.ts @@ -228,7 +228,10 @@ describe("private repository worker defense in depth", () => { expect(exclusiveLock).toContain("lock_timeout = '250ms'"); expect(exclusiveLock).toContain("set_config('lock_timeout', $1, true)"); expect(exclusiveLock).toContain("ROLLBACK TO SAVEPOINT"); - expect(exclusiveLock).toContain("pg_terminate_backend"); + expect(exclusiveLock).not.toContain("pg_terminate_backend"); + expect(rollout).toContain( + "waitForLegacyPublicationLifecycleTransactions(client)", + ); expect(exclusiveLock).toContain("publication lifecycle lock did not quiesce"); expect(activation).toContain("client.release(releaseError)"); expect(activation).not.toContain('query("ROLLBACK").catch'); diff --git a/tests/publication-receipt-migration.test.ts b/tests/publication-receipt-migration.test.ts index ca5ed670..f86966da 100644 --- a/tests/publication-receipt-migration.test.ts +++ b/tests/publication-receipt-migration.test.ts @@ -29,9 +29,11 @@ import { deactivatePublicationLifecycleRelease, prepareManagedReleaseCapabilities, publicationLifecycleReleaseActivated, + recoverAbandonedManagedReleasePreparations, restoreManagedReleaseCapabilities, withPublicationLifecycleReleaseActive, } from "@/lib/release-job-rollout"; +import { compensateReleasePreparation } from "../scripts/run-release-migrations"; const realAppAuth = await import("@/lib/github/app-auth"); const realChecks = await import("@/lib/github/checks"); @@ -783,16 +785,13 @@ describeDb("publication receipt migration and lifecycle", () => { } }); - test("deactivation retires an idle transaction-pool backend and its leaked session state", async () => { + test("deactivation ignores an idle legacy session lock without terminating its backend", async () => { const stalePool = new Pool({ connectionString: TEST_URL, max: 1, application_name: "Supavisor", }); const holder = await stalePool.connect(); - const holderFailure = new Promise((resolve) => { - holder.on("error", resolve); - }); try { await holder.query( "SELECT pg_advisory_lock_shared(hashtextextended($1, 0))", @@ -806,13 +805,6 @@ describeDb("publication receipt migration and lifecycle", () => { expect(await deactivatePublicationLifecycleRelease(pool)).toMatchObject({ deactivated: true, }); - const termination = await Promise.race([ - holderFailure, - Bun.sleep(1_000).then(() => null), - ]); - expect(termination?.message).toContain( - "terminating connection due to administrator command", - ); const leakedState = await pool.query<{ count: string }>( `SELECT count(*)::text AS count FROM pg_locks @@ -823,7 +815,8 @@ describeDb("publication receipt migration and lifecycle", () => { AND objid::bigint = (hashtextextended($1, 0) & 4294967295)`, ["postil:test-leaked-session-state"], ); - expect(leakedState.rows[0]?.count).toBe("0"); + expect(leakedState.rows[0]?.count).toBe("1"); + expect((await holder.query("SELECT 1 AS alive")).rows[0]?.alive).toBe(1); } finally { await holder .query( @@ -837,6 +830,34 @@ describeDb("publication receipt migration and lifecycle", () => { } }); + test("deactivation drains an active legacy transaction before crossing the versioned boundary", async () => { + const legacyPool = new Pool({ connectionString: TEST_URL, max: 1 }); + const holder = await legacyPool.connect(); + try { + await holder.query("BEGIN"); + await holder.query( + "SELECT pg_advisory_xact_lock_shared(hashtextextended($1, 0))", + ["postil:publication-lifecycle-release"], + ); + let finished = false; + const deactivation = deactivatePublicationLifecycleRelease(pool).then( + (result) => { + finished = true; + return result; + }, + ); + await Bun.sleep(150); + expect(finished).toBe(false); + await holder.query("COMMIT"); + await expect(deactivation).resolves.toMatchObject({ deactivated: true }); + } finally { + await holder.query("ROLLBACK").catch(() => undefined); + holder.release(); + await legacyPool.end(); + await activatePublicationLifecycleRelease(pool); + } + }); + test("failed release preparation restores the exact fleet capabilities", async () => { const releaseSha = "8".repeat(40); const capabilityNames = [ @@ -884,6 +905,15 @@ describeDb("publication receipt migration and lifecycle", () => { ).rows[0]?.parked, ).toBe(true); + const parkedHosted = await pool.query<{ id: string }>( + `INSERT INTO jobs (kind, payload, run_after) + VALUES + ('review', jsonb_build_object('releaseDarkSha', $1::text), 'infinity'::timestamptz), + ('hosted-provider-key-lifecycle', jsonb_build_object('releaseDarkSha', $1::text), 'infinity'::timestamptz) + RETURNING id`, + [releaseSha], + ); + await restoreManagedReleaseCapabilities(pool, snapshot); expect( ( @@ -907,6 +937,111 @@ describeDb("publication receipt migration and lifecycle", () => { ) ).rows[0], ).toEqual({ due: true, dark: false }); + const restoredHosted = await pool.query<{ due: boolean; dark: boolean }>( + `SELECT run_after <= now() AS due, + payload ? 'releaseDarkSha' AS dark + FROM jobs + WHERE id = ANY($1::bigint[]) + ORDER BY id`, + [parkedHosted.rows.map((row) => row.id)], + ); + expect(restoredHosted.rows).toEqual([ + { due: true, dark: false }, + { due: true, dark: false }, + ]); + const journal = await pool.query<{ count: string }>( + `SELECT count(*)::text AS count + FROM deployment_capabilities + WHERE name LIKE $1`, + [`managed-release-preparation:${releaseSha}:%`], + ); + expect(journal.rows[0]?.count).toBe("0"); + }); + + test("an old worker restores an abandoned durable preparation but the target release does not", async () => { + const releaseSha = "9".repeat(40); + await pool.query( + `INSERT INTO deployment_capabilities (name) + VALUES ('publication-lifecycle-fleet-active'), + ('hosted-inference-fleet-active'), + ($1) + ON CONFLICT (name) DO NOTHING`, + [`hosted-inference-release:${releaseSha}`], + ); + await prepareManagedReleaseCapabilities(pool, releaseSha, true); + await pool.query( + `UPDATE deployment_capabilities + SET activated_at = now() - interval '30 minutes' + WHERE name LIKE $1`, + [`managed-release-preparation:${releaseSha}:%`], + ); + + expect( + await recoverAbandonedManagedReleasePreparations(pool, releaseSha, 0), + ).toBe(0); + expect( + await recoverAbandonedManagedReleasePreparations( + pool, + "a".repeat(40), + 0, + ), + ).toBe(1); + const restored = await pool.query<{ name: string }>( + `SELECT name FROM deployment_capabilities + WHERE name = ANY($1::text[]) + ORDER BY name`, + [[ + "publication-lifecycle-fleet-active", + "hosted-inference-fleet-active", + `hosted-inference-release:${releaseSha}`, + ]], + ); + expect(restored.rows.map((row) => row.name)).toEqual([ + "hosted-inference-fleet-active", + `hosted-inference-release:${releaseSha}`, + "publication-lifecycle-fleet-active", + ]); + }); + + test("the deploy recovery command restores the exact durable preparation", async () => { + const releaseSha = "7".repeat(40); + await pool.query( + `INSERT INTO deployment_capabilities (name) + VALUES ('publication-lifecycle-fleet-active'), + ('hosted-inference-fleet-active'), + ($1) + ON CONFLICT (name) DO NOTHING`, + [`hosted-inference-release:${releaseSha}`], + ); + await prepareManagedReleaseCapabilities(pool, releaseSha, true); + + expect( + await compensateReleasePreparation({ + DATABASE_URL: TEST_URL!, + POSTIL_RELEASE_SHA: releaseSha, + }), + ).toBe(true); + expect( + await compensateReleasePreparation({ + DATABASE_URL: TEST_URL!, + POSTIL_RELEASE_SHA: releaseSha, + }), + ).toBe(false); + const restored = await pool.query<{ name: string }>( + `SELECT name FROM deployment_capabilities + WHERE name = ANY($1::text[]) + ORDER BY name`, + [[ + "publication-lifecycle-fleet-active", + "hosted-inference-fleet-active", + `hosted-inference-release:${releaseSha}`, + ]], + ); + expect(restored.rows.map((row) => row.name)).toEqual([ + "hosted-inference-fleet-active", + `hosted-inference-release:${releaseSha}`, + "publication-lifecycle-fleet-active", + ]); }); test("a gate committed after the activation sweep self-heals", async () => { @@ -917,7 +1052,7 @@ describeDb("publication receipt migration and lifecycle", () => { await activationClient.query("BEGIN"); await activationClient.query( "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", - ["postil:publication-lifecycle-release"], + ["postil:publication-lifecycle-release-v2"], ); await activationClient.query( `INSERT INTO deployment_capabilities (name) diff --git a/tests/release-database-url.test.ts b/tests/release-database-url.test.ts index a0e7d506..272fc6f9 100644 --- a/tests/release-database-url.test.ts +++ b/tests/release-database-url.test.ts @@ -181,6 +181,46 @@ describe("release database connection", () => { expect(restored).toEqual([snapshot]); }); + test("aborts the active migration child and compensates on termination", async () => { + const environment = { + DATABASE_URL: "postgresql://postil@db.internal:5432/postil", + POSTIL_RELEASE_SHA: "b".repeat(40), + }; + const snapshot = { + releaseSha: "b".repeat(40), + publicationLifecycleReady: true, + capabilities: ["publication-lifecycle-fleet-active"], + }; + const controller = new AbortController(); + let childStarted!: () => void; + const started = new Promise((resolve) => { + childStarted = resolve; + }); + const kills: Array = []; + const restored: unknown[] = []; + const run = runReleaseMigrations( + environment, + () => { + childStarted(); + return { + exited: new Promise(() => undefined), + kill: (signal) => kills.push(signal), + }; + }, + async () => snapshot, + async (_databaseEnvironment, captured) => { + restored.push(captured); + }, + controller.signal, + ); + await started; + controller.abort(); + + await expect(run).rejects.toThrow("release database migration interrupted"); + expect(kills).toEqual(["SIGTERM"]); + expect(restored).toEqual([snapshot]); + }); + test("keeps the checked-in release and deploy contracts aligned", async () => { const root = join(import.meta.dir, ".."); const packageJson = JSON.parse(await readFile(join(root, "package.json"), "utf8")) as { diff --git a/tests/worker-runner.test.ts b/tests/worker-runner.test.ts index 9e5a413f..a69129d1 100644 --- a/tests/worker-runner.test.ts +++ b/tests/worker-runner.test.ts @@ -540,6 +540,8 @@ describe("drainQueueOnce", () => { expect(worker).toContain("activeControllers.set(job.id, controller)"); expect(worker).toContain("requeueableReviewIds.add(job.id)"); expect(worker).toContain("requeueableReviewIds.delete(job.id)"); + expect(worker).toContain("recoverAbandonedManagedReleasePreparations("); + expect(worker).toContain('optionalEnv("POSTIL_RELEASE_SHA")'); // A claim that lands during shutdown is requeued before it is started. expect(worker).toContain('if (shuttingDown && outcome.status === "claimed")'); expect(worker).toContain("[outcome.job.id]"); From 14611f4b7d721f2a6ea2f4465ac31adf081c4ba1 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 01:59:06 +0000 Subject: [PATCH 21/34] Close release recovery races --- .github/workflows/deploy.yml | 11 ++ .github/workflows/production-monitor.yml | 34 ++++- scripts/run-release-migrations.ts | 36 ++++-- src/lib/release-job-rollout.ts | 136 +++++++------------- src/worker/index.ts | 11 -- tests/private-worker-gates.test.ts | 5 +- tests/publication-receipt-migration.test.ts | 111 ++++++++++------ tests/release-database-url.test.ts | 57 +++++++- tests/worker-runner.test.ts | 2 - 9 files changed, 238 insertions(+), 165 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f113cef2..5b0c706d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -333,6 +333,7 @@ jobs: set -euo pipefail machines=$(flyctl machine list --app postil-web --json) target_seen=0 + releases=() while IFS= read -r id; do release=$(flyctl machine exec "${id}" \ "bun -e 'process.stdout.write(process.env.POSTIL_RELEASE_SHA ?? \"\")'" \ @@ -341,6 +342,7 @@ jobs: echo "A managed machine did not report a valid release; capabilities remain dark." exit 1 fi + releases+=("${release}") if [[ "${release}" == "${GITHUB_SHA}" ]]; then target_seen=1 break @@ -355,6 +357,15 @@ jobs: echo "A target-release machine is running; release capabilities remain dark." exit 0 fi + if [[ "${#releases[@]}" -lt 4 ]]; then + echo "Managed fleet evidence is incomplete; release capabilities remain dark." + exit 1 + fi + unique_release_count=$(printf '%s\n' "${releases[@]}" | sort -u | wc -l) + if [[ "${unique_release_count}" -ne 1 ]]; then + echo "The managed fleet is mixed; release capabilities remain dark." + exit 1 + fi bun scripts/run-release-migrations.ts --compensate env: FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} diff --git a/.github/workflows/production-monitor.yml b/.github/workflows/production-monitor.yml index c8e578b4..df06f9e8 100644 --- a/.github/workflows/production-monitor.yml +++ b/.github/workflows/production-monitor.yml @@ -30,6 +30,12 @@ jobs: release-recovery: name: Recover abandoned release preparation if: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.conclusion != 'success' }} + concurrency: + group: fly-deploy + cancel-in-progress: false + permissions: + actions: read + contents: read runs-on: ubuntu-latest timeout-minutes: 6 steps: @@ -61,9 +67,18 @@ jobs: env: FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} DATABASE_URL: ${{ secrets.DATABASE_URL }} + FAILED_DEPLOY_RUN_ID: ${{ github.event.workflow_run.id }} + GH_TOKEN: ${{ github.token }} POSTIL_RELEASE_SHA: ${{ github.event.workflow_run.head_sha }} run: | set -euo pipefail + latest_deploy_run_id=$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/deploy.yml/runs?per_page=1" \ + --jq '.workflow_runs[0].id') + if [[ "${latest_deploy_run_id}" != "${FAILED_DEPLOY_RUN_ID}" ]]; then + echo "A newer deployment owns release recovery." + exit 0 + fi machines=$(flyctl machine list --app postil-web --json) managed_count=$(jq -r '[.[] | select( .config.metadata.fly_process_group == "web" or @@ -80,6 +95,7 @@ jobs: echo "Managed fleet state is incomplete; release capabilities remain dark." exit 1 fi + releases=() while IFS= read -r id; do release=$(flyctl machine exec "${id}" \ "bun -e 'process.stdout.write(process.env.POSTIL_RELEASE_SHA ?? \"\")'" \ @@ -88,6 +104,7 @@ jobs: echo "A managed machine did not report a valid release; capabilities remain dark." exit 1 fi + releases+=("${release}") if [[ "${release}" == "${POSTIL_RELEASE_SHA}" ]]; then echo "The target release reached the managed fleet; capabilities remain dark." exit 1 @@ -98,6 +115,11 @@ jobs: .config.metadata.fly_process_group == "worker" or .config.metadata.fly_process_group == "monitor") ) | .id' <<<"${machines}") + unique_release_count=$(printf '%s\n' "${releases[@]}" | sort -u | wc -l) + if [[ "${#releases[@]}" -ne "${managed_count}" || "${unique_release_count}" -ne 1 ]]; then + echo "The managed fleet is mixed; release capabilities remain dark." + exit 1 + fi bun scripts/run-release-migrations.ts --compensate smoke: @@ -472,8 +494,8 @@ jobs: # runs into one alert, and the resolve job auto-closes it on recovery. notify: name: Raise external alert - needs: smoke - if: ${{ always() && (needs.smoke.result == 'failure' || inputs.test_alert == true) }} + needs: [smoke, release-recovery] + if: ${{ always() && (needs.smoke.result == 'failure' || needs.release-recovery.result == 'failure' || inputs.test_alert == true) }} permissions: contents: read id-token: write @@ -494,10 +516,12 @@ jobs: uses: ./.github/actions/ilert-event with: event-type: ALERT - summary: ${{ needs.smoke.result == 'failure' && 'Postil production monitor failed' || 'Postil production monitor test alert' }} - alert-key: ${{ needs.smoke.result == 'failure' && 'postil-production-monitor' || 'postil-production-monitor-test' }} + summary: ${{ needs.release-recovery.result == 'failure' && 'Postil release recovery failed' || needs.smoke.result == 'failure' && 'Postil production monitor failed' || 'Postil production monitor test alert' }} + alert-key: ${{ inputs.test_alert == true && 'postil-production-monitor-test' || 'postil-production-monitor' }} details: >- - ${{ needs.smoke.result == 'failure' + ${{ needs.release-recovery.result == 'failure' + && 'Failed deployment left release recovery unresolved. Run log:' + || needs.smoke.result == 'failure' && 'Production checks failed. Run log:' || 'Operator-requested test alert; production checks passed. Run log:' }} ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} diff --git a/scripts/run-release-migrations.ts b/scripts/run-release-migrations.ts index 3c3e18fb..1f869c2b 100644 --- a/scripts/run-release-migrations.ts +++ b/scripts/run-release-migrations.ts @@ -25,6 +25,10 @@ type RestoreReleaseCapabilities = ( snapshot: ManagedReleaseCapabilitySnapshot, ) => Promise; +class ReleaseCommandStateUncertainError extends Error { + override name = "ReleaseCommandStateUncertainError"; +} + export function releaseMigrationEnvironment(environment: Environment): Environment { const { POSTIL_DIRECT_DATABASE_URL: directDatabaseUrl, ...migrationEnvironment } = environment; return { @@ -68,7 +72,7 @@ export async function runReleaseMigrations( signal, ); } catch (error) { - if (snapshot) { + if (snapshot && !(error instanceof ReleaseCommandStateUncertainError)) { try { await restoreCapabilities(databaseEnvironment, snapshot); } catch (restoreError) { @@ -173,24 +177,28 @@ async function runReleaseDatabaseCommand( let exitCode: number; let abortHandler: (() => void) | undefined; + let forceKillTimer: ReturnType | undefined; + let interrupted = false; try { - const interrupted = new Promise((_resolve, reject) => { - abortHandler = () => { - process.kill?.("SIGTERM"); - reject(new Error(`${label} interrupted`)); - }; - if (signal?.aborted) abortHandler(); - else signal?.addEventListener("abort", abortHandler, { once: true }); - }); - exitCode = await Promise.race([process.exited, interrupted]); + abortHandler = () => { + if (interrupted) return; + interrupted = true; + process.kill?.("SIGTERM"); + forceKillTimer = setTimeout(() => process.kill?.("SIGKILL"), 10_000); + }; + if (signal?.aborted) abortHandler(); + else signal?.addEventListener("abort", abortHandler, { once: true }); + exitCode = await process.exited; } catch (cause) { - if (cause instanceof Error && cause.message === `${label} interrupted`) { - throw cause; - } - throw new Error(`${label} status could not be observed`, { cause }); + throw new ReleaseCommandStateUncertainError( + `${label} termination could not be observed; durable compensation remains pending`, + { cause }, + ); } finally { + if (forceKillTimer) clearTimeout(forceKillTimer); if (abortHandler) signal?.removeEventListener("abort", abortHandler); } + if (interrupted) throw new Error(`${label} interrupted`); if (exitCode !== 0) { throw new Error(`${label} failed with status ${exitCode}`); } diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index 18520922..62bffb25 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -26,7 +26,6 @@ const PRIVATE_REVIEW_AUTHOR_LOCK = "postil:private-review-author-v1"; const HOSTED_INFERENCE_CAPABILITY_PREFIX = "hosted-inference-release:"; const HOSTED_INFERENCE_DARK_PREFIX = "hosted-inference-dark:"; const MANAGED_RELEASE_PREPARATION_PREFIX = "managed-release-preparation:"; -const MANAGED_RELEASE_RECOVERY_AGE_MS = 20 * 60 * 1000; export const HOSTED_INFERENCE_FLEET_ACTIVE_CAPABILITY = "hosted-inference-fleet-active"; export const HOSTED_INFERENCE_LOCK = "postil:hosted-inference-release"; @@ -35,8 +34,7 @@ export const PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY = const PUBLICATION_LIFECYCLE_DARK_PAYLOAD_KEY = "_postilPublicationLifecycleDark"; const PUBLICATION_LIFECYCLE_LOCK_TIMEOUT_MS = 30_000; -const LEGACY_PUBLICATION_LIFECYCLE_LOCK = - "postil:publication-lifecycle-release"; +const LEGACY_PUBLICATION_DRAIN_TIMEOUT_MS = 120_000; function databaseClientError(error: unknown, fallback: string): Error { return error instanceof Error ? error : new Error(fallback); @@ -102,42 +100,26 @@ async function lockPublicationLifecycleExclusive( } } -async function waitForLegacyPublicationLifecycleTransactions( +async function waitForLegacyPublicationLifecycleOperations( client: PoolClient, ): Promise { - const deadline = Date.now() + PUBLICATION_LIFECYCLE_LOCK_TIMEOUT_MS; + const deadline = Date.now() + LEGACY_PUBLICATION_DRAIN_TIMEOUT_MS; while (true) { - await client.query("SELECT pg_stat_clear_snapshot()"); const active = await client.query<{ active: boolean }>( `SELECT EXISTS ( - SELECT 1 - FROM pg_locks AS advisory - INNER JOIN pg_stat_activity AS activity ON activity.pid = advisory.pid - WHERE advisory.locktype = 'advisory' - AND advisory.granted - AND advisory.mode IN ('ShareLock', 'ExclusiveLock') - AND advisory.objsubid = 1 - AND activity.datname = current_database() - AND activity.usename = current_user - AND advisory.classid::bigint = ( - (hashtextextended($1, 0) >> 32) & 4294967295 - ) - AND advisory.objid::bigint = ( - hashtextextended($1, 0) & 4294967295 - ) - AND advisory.pid <> pg_backend_pid() - AND ( - activity.state IS DISTINCT FROM 'idle' - OR activity.backend_xid IS NOT NULL - OR activity.backend_xmin IS NOT NULL - ) + SELECT 1 FROM jobs + WHERE kind = 'gate-state-sync' + AND status = 'running' + ) OR EXISTS ( + SELECT 1 FROM reviews + WHERE gate_sync_lease_id IS NOT NULL + AND gate_sync_lease_expires_at >= clock_timestamp() ) AS active`, - [LEGACY_PUBLICATION_LIFECYCLE_LOCK], ); if (active.rows[0]?.active !== true) return; if (Date.now() >= deadline) { throw new Error( - "legacy publication lifecycle transactions did not quiesce within 30 seconds", + "legacy publication lifecycle operations did not quiesce within 120 seconds", ); } await client.query("SELECT pg_sleep(0.1)"); @@ -199,9 +181,8 @@ export async function deactivatePublicationLifecycleRelease( let releaseError: Error | undefined; let transactionOpen = false; try { - // Darken first without waiting for the legacy session lock. A pooled - // backend reassigned during cleanup then fails the capability check before - // it can publish. The exclusive pass below drains work already in flight. + // Darken before draining so a legacy publisher that has not passed the + // capability check cannot begin while admitted operations finish. await client.query("BEGIN"); transactionOpen = true; const initial = await darkenPublicationLifecycle(client); @@ -211,7 +192,7 @@ export async function deactivatePublicationLifecycleRelease( await client.query("BEGIN"); transactionOpen = true; await lockPublicationLifecycleExclusive(client); - await waitForLegacyPublicationLifecycleTransactions(client); + await waitForLegacyPublicationLifecycleOperations(client); const fenced = await darkenPublicationLifecycle(client); await client.query("COMMIT"); transactionOpen = false; @@ -640,8 +621,8 @@ export async function activateHostedInferenceRelease( [`${HOSTED_INFERENCE_DARK_PREFIX}%`], ); await client.query( - "DELETE FROM deployment_capabilities WHERE name = ANY($1::text[])", - [Object.values(managedReleasePreparationNames(releaseSha))], + "DELETE FROM deployment_capabilities WHERE name LIKE $1", + [`${MANAGED_RELEASE_PREPARATION_PREFIX}%`], ); await client.query("COMMIT"); return (activated.rowCount ?? 0) > 0; @@ -863,21 +844,12 @@ async function restoreManagedReleaseCapabilitiesInternal( const names = managedReleaseCapabilityNames(snapshot.releaseSha); const journal = managedReleasePreparationNames(snapshot.releaseSha); const journalNames = Object.values(journal); - const expected = new Set(names); - if ( - snapshot.capabilities.some((name) => !expected.has(name)) || - new Set(snapshot.capabilities).size !== snapshot.capabilities.length - ) { - throw new Error("managed release capability snapshot is invalid"); - } - const publicationWasActive = snapshot.capabilities.includes( - PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY, - ); + let effectiveSnapshot = snapshot; const client = await pool.connect(); let releaseError: Error | undefined; try { await client.query("BEGIN"); - if (snapshot.publicationLifecycleReady) { + if (requireJournal || snapshot.publicationLifecycleReady) { await lockPublicationLifecycleExclusive(client); } await client.query( @@ -885,29 +857,43 @@ async function restoreManagedReleaseCapabilitiesInternal( [HOSTED_INFERENCE_LOCK], ); if (requireJournal) { - const durable = await client.query<{ present: boolean }>( - `SELECT EXISTS ( - SELECT 1 FROM deployment_capabilities WHERE name = $1 - ) AS present`, - [journal.root], + const durable = await client.query<{ name: string }>( + "SELECT name FROM deployment_capabilities WHERE name = ANY($1::text[])", + [journalNames], + ); + const current = managedReleasePreparationSnapshot( + snapshot.releaseSha, + durable.rows.map((row) => row.name), ); - if (durable.rows[0]?.present !== true) { + if (!current) { await client.query("COMMIT"); return false; } + effectiveSnapshot = current; } + const expected = new Set(names); + if ( + effectiveSnapshot.capabilities.some((name) => !expected.has(name)) || + new Set(effectiveSnapshot.capabilities).size !== + effectiveSnapshot.capabilities.length + ) { + throw new Error("managed release capability snapshot is invalid"); + } + const publicationWasActive = effectiveSnapshot.capabilities.includes( + PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY, + ); await client.query( "DELETE FROM deployment_capabilities WHERE name = ANY($1::text[])", [names], ); - if (snapshot.capabilities.length > 0) { + if (effectiveSnapshot.capabilities.length > 0) { await client.query( `INSERT INTO deployment_capabilities (name) SELECT unnest($1::text[])`, - [snapshot.capabilities], + [effectiveSnapshot.capabilities], ); } - if (snapshot.publicationLifecycleReady && publicationWasActive) { + if (effectiveSnapshot.publicationLifecycleReady && publicationWasActive) { await client.query( `UPDATE jobs SET run_after = now(), payload = payload - $1 @@ -917,15 +903,19 @@ async function restoreManagedReleaseCapabilitiesInternal( [PUBLICATION_LIFECYCLE_DARK_PAYLOAD_KEY], ); } - if (snapshot.capabilities.includes(HOSTED_INFERENCE_FLEET_ACTIVE_CAPABILITY)) { + if ( + effectiveSnapshot.capabilities.includes( + HOSTED_INFERENCE_FLEET_ACTIVE_CAPABILITY, + ) + ) { await client.query( `UPDATE jobs SET run_after = now(), payload = payload - 'releaseDarkSha' WHERE kind IN ('review', $1) AND status = 'queued' AND run_after = 'infinity'::timestamptz - AND payload->>'releaseDarkSha' = $2`, - [HOSTED_PROVIDER_KEY_LIFECYCLE_JOB_KIND, snapshot.releaseSha], + AND payload ? 'releaseDarkSha'`, + [HOSTED_PROVIDER_KEY_LIFECYCLE_JOB_KIND], ); } await client.query( @@ -977,36 +967,6 @@ export async function restoreManagedReleasePreparation( return restoreManagedReleaseCapabilitiesInternal(pool, snapshot, true); } -/** Restore release preparation abandoned by a dead deploy process. */ -export async function recoverAbandonedManagedReleasePreparations( - pool: Pool, - currentReleaseSha: string | undefined, - minimumAgeMs = MANAGED_RELEASE_RECOVERY_AGE_MS, -): Promise { - const current = currentReleaseSha - ? normalizedReleaseSha(currentReleaseSha) - : undefined; - const roots = await pool.query<{ name: string }>( - `SELECT name - FROM deployment_capabilities - WHERE name LIKE $1 - AND name LIKE '%:root' - AND activated_at <= now() - ($2::double precision * interval '1 millisecond') - ORDER BY activated_at`, - [`${MANAGED_RELEASE_PREPARATION_PREFIX}%`, minimumAgeMs], - ); - let recovered = 0; - for (const row of roots.rows) { - const releaseSha = row.name.slice( - MANAGED_RELEASE_PREPARATION_PREFIX.length, - -":root".length, - ); - if (!/^[0-9a-f]{7,40}$/.test(releaseSha) || releaseSha === current) continue; - if (await restoreManagedReleasePreparation(pool, releaseSha)) recovered += 1; - } - return recovered; -} - /** Atomically park a claimed hosted review until a verified managed release activates. */ export async function deferHostedReviewForRelease( pool: Pool, diff --git a/src/worker/index.ts b/src/worker/index.ts index 27cfd9bb..64a5a539 100644 --- a/src/worker/index.ts +++ b/src/worker/index.ts @@ -21,7 +21,6 @@ import { } from "@/lib/queue"; import { redactSecrets } from "@/lib/redact"; import { recoverRespondDeliveryJobs } from "@/lib/respond-delivery"; -import { recoverAbandonedManagedReleasePreparations } from "@/lib/release-job-rollout"; import { recordServiceHeartbeat } from "@/lib/private-monitoring"; import { configuredPrivateWorkerRehearsalSandbox, @@ -257,16 +256,6 @@ function jitter(delayMs: number): number { async function watchdogLoop(): Promise { while (!shuttingDown) { try { - const recoveredReleasePreparations = - await recoverAbandonedManagedReleasePreparations( - getPool(), - optionalEnv("POSTIL_RELEASE_SHA"), - ); - if (recoveredReleasePreparations > 0) { - console.warn( - `[watchdog] recovered ${recoveredReleasePreparations} abandoned release preparation(s)`, - ); - } await enqueueGateEnforcementSweepOnce(getPool(), { minIntervalMs: GATE_ENFORCEMENT_SWEEP_INTERVAL_MS, }); diff --git a/tests/private-worker-gates.test.ts b/tests/private-worker-gates.test.ts index 55a7cc14..58938e13 100644 --- a/tests/private-worker-gates.test.ts +++ b/tests/private-worker-gates.test.ts @@ -230,8 +230,11 @@ describe("private repository worker defense in depth", () => { expect(exclusiveLock).toContain("ROLLBACK TO SAVEPOINT"); expect(exclusiveLock).not.toContain("pg_terminate_backend"); expect(rollout).toContain( - "waitForLegacyPublicationLifecycleTransactions(client)", + "waitForLegacyPublicationLifecycleOperations(client)", ); + expect(rollout).toContain("kind = 'gate-state-sync'"); + expect(rollout).toContain("status = 'running'"); + expect(exclusiveLock).not.toContain("pg_stat_activity"); expect(exclusiveLock).toContain("publication lifecycle lock did not quiesce"); expect(activation).toContain("client.release(releaseError)"); expect(activation).not.toContain('query("ROLLBACK").catch'); diff --git a/tests/publication-receipt-migration.test.ts b/tests/publication-receipt-migration.test.ts index f86966da..b48ea273 100644 --- a/tests/publication-receipt-migration.test.ts +++ b/tests/publication-receipt-migration.test.ts @@ -29,8 +29,8 @@ import { deactivatePublicationLifecycleRelease, prepareManagedReleaseCapabilities, publicationLifecycleReleaseActivated, - recoverAbandonedManagedReleasePreparations, restoreManagedReleaseCapabilities, + restoreManagedReleasePreparation, withPublicationLifecycleReleaseActive, } from "@/lib/release-job-rollout"; import { compensateReleasePreparation } from "../scripts/run-release-migrations"; @@ -830,7 +830,7 @@ describeDb("publication receipt migration and lifecycle", () => { } }); - test("deactivation drains an active legacy transaction before crossing the versioned boundary", async () => { + test("deactivation drains a durable legacy publication operation without trusting backend state", async () => { const legacyPool = new Pool({ connectionString: TEST_URL, max: 1 }); const holder = await legacyPool.connect(); try { @@ -839,6 +839,14 @@ describeDb("publication receipt migration and lifecycle", () => { "SELECT pg_advisory_xact_lock_shared(hashtextextended($1, 0))", ["postil:publication-lifecycle-release"], ); + const legacyJob = await pool.query<{ id: string }>( + `INSERT INTO jobs + (kind, payload, status, locked_at, locked_by) + VALUES + ('gate-state-sync', '{"reviewId":1,"reviewPublicId":"legacy-drain"}'::jsonb, + 'running', now(), 'legacy-worker') + RETURNING id`, + ); let finished = false; const deactivation = deactivatePublicationLifecycleRelease(pool).then( (result) => { @@ -848,8 +856,15 @@ describeDb("publication receipt migration and lifecycle", () => { ); await Bun.sleep(150); expect(finished).toBe(false); - await holder.query("COMMIT"); + await pool.query( + `UPDATE jobs + SET status = 'done', locked_at = NULL, locked_by = NULL + WHERE id = $1`, + [legacyJob.rows[0]!.id], + ); await expect(deactivation).resolves.toMatchObject({ deactivated: true }); + expect((await holder.query("SELECT 1 AS alive")).rows[0]?.alive).toBe(1); + await holder.query("COMMIT"); } finally { await holder.query("ROLLBACK").catch(() => undefined); holder.release(); @@ -860,6 +875,7 @@ describeDb("publication receipt migration and lifecycle", () => { test("failed release preparation restores the exact fleet capabilities", async () => { const releaseSha = "8".repeat(40); + const priorReleaseSha = "5".repeat(40); const capabilityNames = [ "publication-lifecycle-fleet-active", "hosted-inference-fleet-active", @@ -911,7 +927,7 @@ describeDb("publication receipt migration and lifecycle", () => { ('review', jsonb_build_object('releaseDarkSha', $1::text), 'infinity'::timestamptz), ('hosted-provider-key-lifecycle', jsonb_build_object('releaseDarkSha', $1::text), 'infinity'::timestamptz) RETURNING id`, - [releaseSha], + [priorReleaseSha], ); await restoreManagedReleaseCapabilities(pool, snapshot); @@ -958,8 +974,8 @@ describeDb("publication receipt migration and lifecycle", () => { expect(journal.rows[0]?.count).toBe("0"); }); - test("an old worker restores an abandoned durable preparation but the target release does not", async () => { - const releaseSha = "9".repeat(40); + test("the deploy recovery command restores the exact durable preparation", async () => { + const releaseSha = "7".repeat(40); await pool.query( `INSERT INTO deployment_capabilities (name) VALUES ('publication-lifecycle-fleet-active'), @@ -969,23 +985,19 @@ describeDb("publication receipt migration and lifecycle", () => { [`hosted-inference-release:${releaseSha}`], ); await prepareManagedReleaseCapabilities(pool, releaseSha, true); - await pool.query( - `UPDATE deployment_capabilities - SET activated_at = now() - interval '30 minutes' - WHERE name LIKE $1`, - [`managed-release-preparation:${releaseSha}:%`], - ); expect( - await recoverAbandonedManagedReleasePreparations(pool, releaseSha, 0), - ).toBe(0); + await compensateReleasePreparation({ + DATABASE_URL: TEST_URL!, + POSTIL_RELEASE_SHA: releaseSha, + }), + ).toBe(true); expect( - await recoverAbandonedManagedReleasePreparations( - pool, - "a".repeat(40), - 0, - ), - ).toBe(1); + await compensateReleasePreparation({ + DATABASE_URL: TEST_URL!, + POSTIL_RELEASE_SHA: releaseSha, + }), + ).toBe(false); const restored = await pool.query<{ name: string }>( `SELECT name FROM deployment_capabilities WHERE name = ANY($1::text[]) @@ -1003,45 +1015,62 @@ describeDb("publication receipt migration and lifecycle", () => { ]); }); - test("the deploy recovery command restores the exact durable preparation", async () => { - const releaseSha = "7".repeat(40); + test("same-release recovery re-reads a replacement journal under the lifecycle locks", async () => { + const releaseSha = "6".repeat(40); + const capabilityNames = [ + "publication-lifecycle-fleet-active", + "hosted-inference-fleet-active", + `hosted-inference-release:${releaseSha}`, + `hosted-inference-dark:${releaseSha}`, + ]; + await pool.query( + "DELETE FROM deployment_capabilities WHERE name = ANY($1::text[])", + [capabilityNames], + ); await pool.query( `INSERT INTO deployment_capabilities (name) VALUES ('publication-lifecycle-fleet-active'), ('hosted-inference-fleet-active'), - ($1) - ON CONFLICT (name) DO NOTHING`, + ($1)`, [`hosted-inference-release:${releaseSha}`], ); await prepareManagedReleaseCapabilities(pool, releaseSha, true); + let replaced = false; + const interceptedPool = { + query: async (text: string, values?: readonly unknown[]) => { + const result = await pool.query(text, values as never[] | undefined); + if (!replaced) { + replaced = true; + await restoreManagedReleasePreparation(pool, releaseSha); + await pool.query( + "DELETE FROM deployment_capabilities WHERE name = ANY($1::text[])", + [capabilityNames], + ); + await pool.query( + `INSERT INTO deployment_capabilities (name) + VALUES ('hosted-inference-fleet-active')`, + ); + await prepareManagedReleaseCapabilities(pool, releaseSha, true); + } + return result; + }, + connect: () => pool.connect(), + } as unknown as Pool; + expect( - await compensateReleasePreparation({ - DATABASE_URL: TEST_URL!, - POSTIL_RELEASE_SHA: releaseSha, - }), + await restoreManagedReleasePreparation(interceptedPool, releaseSha), ).toBe(true); - expect( - await compensateReleasePreparation({ - DATABASE_URL: TEST_URL!, - POSTIL_RELEASE_SHA: releaseSha, - }), - ).toBe(false); const restored = await pool.query<{ name: string }>( `SELECT name FROM deployment_capabilities WHERE name = ANY($1::text[]) ORDER BY name`, - [[ - "publication-lifecycle-fleet-active", - "hosted-inference-fleet-active", - `hosted-inference-release:${releaseSha}`, - ]], + [capabilityNames], ); expect(restored.rows.map((row) => row.name)).toEqual([ "hosted-inference-fleet-active", - `hosted-inference-release:${releaseSha}`, - "publication-lifecycle-fleet-active", ]); + await activatePublicationLifecycleRelease(pool); }); test("a gate committed after the activation sweep self-heals", async () => { diff --git a/tests/release-database-url.test.ts b/tests/release-database-url.test.ts index 272fc6f9..cac2960a 100644 --- a/tests/release-database-url.test.ts +++ b/tests/release-database-url.test.ts @@ -138,7 +138,7 @@ describe("release database connection", () => { ).rejects.toThrow("release database migration could not start"); await expect( runReleaseMigrations(environment, () => ({ exited: Promise.reject(new Error("lost child")) })), - ).rejects.toThrow("release database migration status could not be observed"); + ).rejects.toThrow("release database migration termination could not be observed"); }); test("restores the captured capability state when any database preparation step fails", async () => { @@ -198,17 +198,27 @@ describe("release database connection", () => { }); const kills: Array = []; const restored: unknown[] = []; + const events: string[] = []; + let childExited!: (exitCode: number) => void; + const exited = new Promise((resolve) => { + childExited = resolve; + }); const run = runReleaseMigrations( environment, () => { childStarted(); return { - exited: new Promise(() => undefined), - kill: (signal) => kills.push(signal), + exited, + kill: (signal) => { + events.push("child terminated"); + kills.push(signal); + childExited(143); + }, }; }, async () => snapshot, async (_databaseEnvironment, captured) => { + events.push("capabilities restored"); restored.push(captured); }, controller.signal, @@ -219,6 +229,30 @@ describe("release database connection", () => { await expect(run).rejects.toThrow("release database migration interrupted"); expect(kills).toEqual(["SIGTERM"]); expect(restored).toEqual([snapshot]); + expect(events).toEqual(["child terminated", "capabilities restored"]); + }); + + test("leaves durable compensation pending when child termination is unobservable", async () => { + const snapshot = { + releaseSha: "c".repeat(40), + publicationLifecycleReady: true, + capabilities: ["publication-lifecycle-fleet-active"], + }; + const restored: unknown[] = []; + await expect( + runReleaseMigrations( + { + DATABASE_URL: "postgresql://postil@db.internal:5432/postil", + POSTIL_RELEASE_SHA: snapshot.releaseSha, + }, + () => ({ exited: Promise.reject(new Error("lost child")) }), + async () => snapshot, + async (_databaseEnvironment, captured) => { + restored.push(captured); + }, + ), + ).rejects.toThrow("durable compensation remains pending"); + expect(restored).toEqual([]); }); test("keeps the checked-in release and deploy contracts aligned", async () => { @@ -227,6 +261,10 @@ describe("release database connection", () => { scripts: Record; }; const deployWorkflow = await readFile(join(root, ".github", "workflows", "deploy.yml"), "utf8"); + const productionMonitorWorkflow = await readFile( + join(root, ".github", "workflows", "production-monitor.yml"), + "utf8", + ); const deactivationScript = await readFile( join(root, "scripts", "deactivate-hosted-inference.ts"), "utf8", @@ -240,6 +278,19 @@ describe("release database connection", () => { ); expect(deployWorkflow).toContain('staged+="DATABASE_URL=${DATABASE_URL}"'); expect(deployWorkflow).not.toContain("POSTIL_DIRECT_DATABASE_URL"); + expect(deployWorkflow).toContain( + "Restore capabilities when release preparation failed before replacement", + ); + expect(deployWorkflow).toContain("bun scripts/run-release-migrations.ts --compensate"); + expect(productionMonitorWorkflow).toContain('workflows: ["deploy"]'); + expect(productionMonitorWorkflow).toContain("group: fly-deploy"); + expect(productionMonitorWorkflow).toContain("latest_deploy_run_id"); + expect(productionMonitorWorkflow).toContain( + "needs: [smoke, release-recovery]", + ); + expect(productionMonitorWorkflow).toContain( + "Postil release recovery failed", + ); expect(deactivationScript).toContain("resolveDirectDatabaseUrl"); expect(deactivationScript).toContain("publication_lifecycle_required_at"); expect(deactivationScript.indexOf("process.env.DATABASE_URL =")).toBeLessThan( diff --git a/tests/worker-runner.test.ts b/tests/worker-runner.test.ts index a69129d1..9e5a413f 100644 --- a/tests/worker-runner.test.ts +++ b/tests/worker-runner.test.ts @@ -540,8 +540,6 @@ describe("drainQueueOnce", () => { expect(worker).toContain("activeControllers.set(job.id, controller)"); expect(worker).toContain("requeueableReviewIds.add(job.id)"); expect(worker).toContain("requeueableReviewIds.delete(job.id)"); - expect(worker).toContain("recoverAbandonedManagedReleasePreparations("); - expect(worker).toContain('optionalEnv("POSTIL_RELEASE_SHA")'); // A claim that lands during shutdown is requeued before it is started. expect(worker).toContain('if (shuttingDown && outcome.status === "claimed")'); expect(worker).toContain("[outcome.job.id]"); From 3acfe9f7593c27ccffcf9e2f6641d7e28d01dfc0 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 02:15:22 +0000 Subject: [PATCH 22/34] Fence release preparation generations --- .github/workflows/deploy.yml | 26 +++-- .github/workflows/production-monitor.yml | 54 ++++++++-- scripts/run-release-migrations.ts | 42 +++++++- src/lib/release-job-rollout.ts | 96 ++++++++++++----- tests/publication-receipt-migration.test.ts | 111 +++++++++++++++----- tests/release-database-url.test.ts | 7 ++ 6 files changed, 263 insertions(+), 73 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5b0c706d..e72f71be 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -332,6 +332,21 @@ jobs: run: | set -euo pipefail machines=$(flyctl machine list --app postil-web --json) + managed_count=$(jq -r '[.[] | select( + .config.metadata.fly_process_group == "web" or + .config.metadata.fly_process_group == "worker" or + .config.metadata.fly_process_group == "monitor" + )] | length' <<<"${machines}") + started_count=$(jq -r '[.[] | select( + .state == "started" and + (.config.metadata.fly_process_group == "web" or + .config.metadata.fly_process_group == "worker" or + .config.metadata.fly_process_group == "monitor") + )] | length' <<<"${machines}") + if [[ "${managed_count}" -lt 4 || "${started_count}" -ne "${managed_count}" ]]; then + echo "Managed fleet state is incomplete; release capabilities remain dark." + exit 1 + fi target_seen=0 releases=() while IFS= read -r id; do @@ -345,7 +360,6 @@ jobs: releases+=("${release}") if [[ "${release}" == "${GITHUB_SHA}" ]]; then target_seen=1 - break fi done < <(jq -r '.[] | select( .state == "started" and @@ -353,11 +367,7 @@ jobs: .config.metadata.fly_process_group == "worker" or .config.metadata.fly_process_group == "monitor") ) | .id' <<<"${machines}") - if [[ "${target_seen}" -ne 0 ]]; then - echo "A target-release machine is running; release capabilities remain dark." - exit 0 - fi - if [[ "${#releases[@]}" -lt 4 ]]; then + if [[ "${#releases[@]}" -ne "${managed_count}" ]]; then echo "Managed fleet evidence is incomplete; release capabilities remain dark." exit 1 fi @@ -366,6 +376,10 @@ jobs: echo "The managed fleet is mixed; release capabilities remain dark." exit 1 fi + if [[ "${target_seen}" -ne 0 ]]; then + echo "The target release reached the managed fleet; release capabilities remain dark." + exit 1 + fi bun scripts/run-release-migrations.ts --compensate env: FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} diff --git a/.github/workflows/production-monitor.yml b/.github/workflows/production-monitor.yml index df06f9e8..613722e7 100644 --- a/.github/workflows/production-monitor.yml +++ b/.github/workflows/production-monitor.yml @@ -96,6 +96,7 @@ jobs: exit 1 fi releases=() + target_seen=0 while IFS= read -r id; do release=$(flyctl machine exec "${id}" \ "bun -e 'process.stdout.write(process.env.POSTIL_RELEASE_SHA ?? \"\")'" \ @@ -106,8 +107,7 @@ jobs: fi releases+=("${release}") if [[ "${release}" == "${POSTIL_RELEASE_SHA}" ]]; then - echo "The target release reached the managed fleet; capabilities remain dark." - exit 1 + target_seen=1 fi done < <(jq -r '.[] | select( .state == "started" and @@ -120,6 +120,10 @@ jobs: echo "The managed fleet is mixed; release capabilities remain dark." exit 1 fi + if [[ "${target_seen}" -ne 0 ]]; then + echo "The target release reached the managed fleet; capabilities remain dark." + exit 1 + fi bun scripts/run-release-migrations.ts --compensate smoke: @@ -517,7 +521,7 @@ jobs: with: event-type: ALERT summary: ${{ needs.release-recovery.result == 'failure' && 'Postil release recovery failed' || needs.smoke.result == 'failure' && 'Postil production monitor failed' || 'Postil production monitor test alert' }} - alert-key: ${{ inputs.test_alert == true && 'postil-production-monitor-test' || 'postil-production-monitor' }} + alert-key: ${{ needs.release-recovery.result == 'failure' && 'postil-release-recovery' || inputs.test_alert == true && 'postil-production-monitor-test' || 'postil-production-monitor' }} details: >- ${{ needs.release-recovery.result == 'failure' && 'Failed deployment left release recovery unresolved. Run log:' @@ -525,11 +529,45 @@ jobs: && 'Production checks failed. Run log:' || 'Operator-requested test alert; production checks passed. Run log:' }} ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} - # A production failure pages if it can and records the gap if it - # cannot, because the failing check is already the signal. A test - # alert exists only to prove delivery works, so an undelivered one is - # the failure it was run to detect. - require-delivery: ${{ inputs.test_alert == true }} + # A routine monitor failure records an alerting gap without masking + # the original signal. Recovery failure and test events require + # delivery because they validate the fail-safe notification path. + require-delivery: ${{ inputs.test_alert == true || needs.release-recovery.result == 'failure' }} + + resolve-release-recovery: + name: Resolve release recovery alert + if: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' }} + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + timeout-minutes: 3 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.3.14 + - name: Install dependencies + run: bun install --frozen-lockfile + - name: Verify release state is active and clear + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + run: bun scripts/run-release-migrations.ts --verify-clear + - name: Load alerting secret from Infisical + uses: Infisical/secrets-action@77ab1f4ccd183a543cb5b42435fbd181189f4995 # v1.0.16 + with: + method: oidc + identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} + project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }} + env-slug: prod + domain: https://eu.infisical.com + secret-path: /postil + - name: Resolve ilert release recovery alert + uses: ./.github/actions/ilert-event + with: + event-type: RESOLVE + summary: Postil release recovery cleared + alert-key: postil-release-recovery resolve: name: Resolve external alert diff --git a/scripts/run-release-migrations.ts b/scripts/run-release-migrations.ts index 1f869c2b..f05c11dd 100644 --- a/scripts/run-release-migrations.ts +++ b/scripts/run-release-migrations.ts @@ -3,7 +3,7 @@ import { Pool } from "pg"; import { type ManagedReleaseCapabilitySnapshot, prepareManagedReleaseCapabilities, - restoreManagedReleasePreparation, + restoreAllManagedReleasePreparations, restoreManagedReleaseCapabilities, } from "@/lib/release-job-rollout"; import { resolveDirectDatabaseUrl } from "./resolve-direct-database-url"; @@ -155,7 +155,38 @@ export async function compensateReleasePreparation( try { const schema = await releaseSchemaState(pool); if (!schema.hostedReady) return false; - return await restoreManagedReleasePreparation(pool, releaseSha); + return (await restoreAllManagedReleasePreparations(pool)) > 0; + } finally { + await pool.end(); + } +} + +export async function releasePreparationCleared( + environment: Environment = process.env, +): Promise { + const databaseEnvironment = releaseMigrationEnvironment(environment); + const pool = new Pool({ connectionString: databaseEnvironment.DATABASE_URL }); + try { + const schema = await releaseSchemaState(pool); + if (!schema.publicationLifecycleReady) return false; + const state = await pool.query<{ ready: boolean }>( + `SELECT + NOT EXISTS ( + SELECT 1 FROM deployment_capabilities WHERE name LIKE $1 + ) + AND EXISTS ( + SELECT 1 FROM deployment_capabilities WHERE name = $2 + ) + AND EXISTS ( + SELECT 1 FROM deployment_capabilities WHERE name = $3 + ) AS ready`, + [ + "managed-release-preparation:%", + "publication-lifecycle-fleet-active", + "hosted-inference-fleet-active", + ], + ); + return state.rows[0]?.ready === true; } finally { await pool.end(); } @@ -224,6 +255,13 @@ if (import.meta.main) { ); process.exit(0); } + if (process.argv[2] === "--verify-clear") { + if (!(await releasePreparationCleared())) { + throw new Error("release preparation remains pending or fleet capabilities are dark"); + } + console.log("release preparation state is clear and active"); + process.exit(0); + } const controller = new AbortController(); const interrupt = (signal: NodeJS.Signals) => { controller.abort(new Error(`release database preparation received ${signal}`)); diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index 62bffb25..f40f1c39 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -1,3 +1,5 @@ +import { randomUUID } from "node:crypto"; + import type { Pool, PoolClient } from "pg"; import type { Database } from "@/lib/db"; @@ -673,6 +675,7 @@ export async function deactivateHostedInferenceRelease( export interface ManagedReleaseCapabilitySnapshot { releaseSha: string; + generation: string; publicationLifecycleReady: boolean; capabilities: string[]; } @@ -686,7 +689,10 @@ function managedReleaseCapabilityNames(releaseSha: string): string[] { ]; } -function managedReleasePreparationNames(releaseSha: string): { +function managedReleasePreparationNames( + releaseSha: string, + generation: string, +): { root: string; publicationReady: string; publicationActive: string; @@ -694,7 +700,7 @@ function managedReleasePreparationNames(releaseSha: string): { hostedReleaseActive: string; hostedDarkActive: string; } { - const prefix = `${MANAGED_RELEASE_PREPARATION_PREFIX}${releaseSha}:`; + const prefix = `${MANAGED_RELEASE_PREPARATION_PREFIX}${releaseSha}:${generation}:`; return { root: `${prefix}root`, publicationReady: `${prefix}publication-ready`, @@ -707,9 +713,10 @@ function managedReleasePreparationNames(releaseSha: string): { function managedReleasePreparationSnapshot( releaseSha: string, + generation: string, names: readonly string[], ): ManagedReleaseCapabilitySnapshot | undefined { - const journal = managedReleasePreparationNames(releaseSha); + const journal = managedReleasePreparationNames(releaseSha, generation); const present = new Set(names); if (!present.has(journal.root)) return undefined; const capabilities: string[] = []; @@ -727,6 +734,7 @@ function managedReleasePreparationSnapshot( } return { releaseSha, + generation, publicationLifecycleReady: present.has(journal.publicationReady), capabilities, }; @@ -738,7 +746,8 @@ async function captureManagedReleaseCapabilities( publicationLifecycleReady: boolean, ): Promise { const names = managedReleaseCapabilityNames(releaseSha); - const journal = managedReleasePreparationNames(releaseSha); + const generation = randomUUID(); + const journal = managedReleasePreparationNames(releaseSha, generation); const client = await pool.connect(); try { await client.query("BEGIN"); @@ -755,6 +764,7 @@ async function captureManagedReleaseCapabilities( ); const snapshot: ManagedReleaseCapabilitySnapshot = { releaseSha, + generation, publicationLifecycleReady, capabilities: existing.rows.map((row) => row.name), }; @@ -797,7 +807,7 @@ export async function prepareManagedReleaseCapabilities( publicationLifecycleReady: boolean, ): Promise { const normalizedRelease = normalizedReleaseSha(releaseSha); - await restoreManagedReleasePreparation(pool, normalizedRelease); + await restoreAllManagedReleasePreparations(pool); const snapshot = await captureManagedReleaseCapabilities( pool, normalizedRelease, @@ -833,43 +843,40 @@ export async function restoreManagedReleaseCapabilities( pool: Pool, snapshot: ManagedReleaseCapabilitySnapshot, ): Promise { - await restoreManagedReleaseCapabilitiesInternal(pool, snapshot, false); + await restoreManagedReleaseCapabilitiesInternal(pool, snapshot); } async function restoreManagedReleaseCapabilitiesInternal( pool: Pool, snapshot: ManagedReleaseCapabilitySnapshot, - requireJournal: boolean, ): Promise { const names = managedReleaseCapabilityNames(snapshot.releaseSha); - const journal = managedReleasePreparationNames(snapshot.releaseSha); + const journal = managedReleasePreparationNames( + snapshot.releaseSha, + snapshot.generation, + ); const journalNames = Object.values(journal); - let effectiveSnapshot = snapshot; const client = await pool.connect(); let releaseError: Error | undefined; try { await client.query("BEGIN"); - if (requireJournal || snapshot.publicationLifecycleReady) { - await lockPublicationLifecycleExclusive(client); - } + await lockPublicationLifecycleExclusive(client); await client.query( "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [HOSTED_INFERENCE_LOCK], ); - if (requireJournal) { - const durable = await client.query<{ name: string }>( - "SELECT name FROM deployment_capabilities WHERE name = ANY($1::text[])", - [journalNames], - ); - const current = managedReleasePreparationSnapshot( - snapshot.releaseSha, - durable.rows.map((row) => row.name), - ); - if (!current) { - await client.query("COMMIT"); - return false; - } - effectiveSnapshot = current; + const durable = await client.query<{ name: string }>( + "SELECT name FROM deployment_capabilities WHERE name = ANY($1::text[])", + [journalNames], + ); + const effectiveSnapshot = managedReleasePreparationSnapshot( + snapshot.releaseSha, + snapshot.generation, + durable.rows.map((row) => row.name), + ); + if (!effectiveSnapshot) { + await client.query("COMMIT"); + return false; } const expected = new Set(names); if ( @@ -952,19 +959,52 @@ async function restoreManagedReleaseCapabilitiesInternal( export async function restoreManagedReleasePreparation( pool: Pool, releaseSha: string, + generation: string, ): Promise { const normalizedRelease = normalizedReleaseSha(releaseSha); - const journal = managedReleasePreparationNames(normalizedRelease); + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(generation)) { + throw new Error("managed release preparation generation is invalid"); + } + const journal = managedReleasePreparationNames( + normalizedRelease, + generation, + ); const durable = await pool.query<{ name: string }>( "SELECT name FROM deployment_capabilities WHERE name = ANY($1::text[])", [Object.values(journal)], ); const snapshot = managedReleasePreparationSnapshot( normalizedRelease, + generation, durable.rows.map((row) => row.name), ); if (!snapshot) return false; - return restoreManagedReleaseCapabilitiesInternal(pool, snapshot, true); + return restoreManagedReleaseCapabilitiesInternal(pool, snapshot); +} + +/** Unwind every pending preparation from newest to oldest. */ +export async function restoreAllManagedReleasePreparations( + pool: Pool, +): Promise { + const roots = await pool.query<{ name: string }>( + `SELECT name + FROM deployment_capabilities + WHERE name LIKE $1 + AND name LIKE '%:root' + ORDER BY activated_at DESC, name DESC`, + [`${MANAGED_RELEASE_PREPARATION_PREFIX}%`], + ); + let restored = 0; + for (const row of roots.rows) { + const match = row.name.match( + /^managed-release-preparation:([0-9a-f]{7,40}):([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}):root$/, + ); + if (!match) continue; + if (await restoreManagedReleasePreparation(pool, match[1]!, match[2]!)) { + restored += 1; + } + } + return restored; } /** Atomically park a claimed hosted review until a verified managed release activates. */ diff --git a/tests/publication-receipt-migration.test.ts b/tests/publication-receipt-migration.test.ts index b48ea273..56888967 100644 --- a/tests/publication-receipt-migration.test.ts +++ b/tests/publication-receipt-migration.test.ts @@ -33,7 +33,10 @@ import { restoreManagedReleasePreparation, withPublicationLifecycleReleaseActive, } from "@/lib/release-job-rollout"; -import { compensateReleasePreparation } from "../scripts/run-release-migrations"; +import { + compensateReleasePreparation, + releasePreparationCleared, +} from "../scripts/run-release-migrations"; const realAppAuth = await import("@/lib/github/app-auth"); const realChecks = await import("@/lib/github/checks"); @@ -986,6 +989,10 @@ describeDb("publication receipt migration and lifecycle", () => { ); await prepareManagedReleaseCapabilities(pool, releaseSha, true); + expect( + await releasePreparationCleared({ DATABASE_URL: TEST_URL! }), + ).toBe(false); + expect( await compensateReleasePreparation({ DATABASE_URL: TEST_URL!, @@ -1013,9 +1020,12 @@ describeDb("publication receipt migration and lifecycle", () => { `hosted-inference-release:${releaseSha}`, "publication-lifecycle-fleet-active", ]); + expect( + await releasePreparationCleared({ DATABASE_URL: TEST_URL! }), + ).toBe(true); }); - test("same-release recovery re-reads a replacement journal under the lifecycle locks", async () => { + test("same-release process compensation cannot overwrite a replacement generation", async () => { const releaseSha = "6".repeat(40); const capabilityNames = [ "publication-lifecycle-fleet-active", @@ -1034,42 +1044,85 @@ describeDb("publication receipt migration and lifecycle", () => { ($1)`, [`hosted-inference-release:${releaseSha}`], ); - await prepareManagedReleaseCapabilities(pool, releaseSha, true); - - let replaced = false; - const interceptedPool = { - query: async (text: string, values?: readonly unknown[]) => { - const result = await pool.query(text, values as never[] | undefined); - if (!replaced) { - replaced = true; - await restoreManagedReleasePreparation(pool, releaseSha); - await pool.query( - "DELETE FROM deployment_capabilities WHERE name = ANY($1::text[])", - [capabilityNames], - ); - await pool.query( - `INSERT INTO deployment_capabilities (name) - VALUES ('hosted-inference-fleet-active')`, - ); - await prepareManagedReleaseCapabilities(pool, releaseSha, true); - } - return result; - }, - connect: () => pool.connect(), - } as unknown as Pool; - + const original = await prepareManagedReleaseCapabilities( + pool, + releaseSha, + true, + ); expect( - await restoreManagedReleasePreparation(interceptedPool, releaseSha), + await restoreManagedReleasePreparation( + pool, + releaseSha, + original.generation, + ), ).toBe(true); - const restored = await pool.query<{ name: string }>( + await pool.query( + "DELETE FROM deployment_capabilities WHERE name = ANY($1::text[])", + [capabilityNames], + ); + await pool.query( + `INSERT INTO deployment_capabilities (name) + VALUES ('hosted-inference-fleet-active')`, + ); + const replacement = await prepareManagedReleaseCapabilities( + pool, + releaseSha, + true, + ); + + await restoreManagedReleaseCapabilities(pool, original); + const stillDark = await pool.query<{ name: string }>( `SELECT name FROM deployment_capabilities WHERE name = ANY($1::text[]) ORDER BY name`, [capabilityNames], ); - expect(restored.rows.map((row) => row.name)).toEqual([ + expect(stillDark.rows.map((row) => row.name)).toEqual([ + `hosted-inference-dark:${releaseSha}`, + ]); + expect( + await restoreManagedReleasePreparation( + pool, + releaseSha, + replacement.generation, + ), + ).toBe(true); + await activatePublicationLifecycleRelease(pool); + }); + + test("a newer preparation adopts and replaces every superseded journal", async () => { + const firstRelease = "4".repeat(40); + const secondRelease = "3".repeat(40); + await pool.query( + `INSERT INTO deployment_capabilities (name) + VALUES ('publication-lifecycle-fleet-active'), + ('hosted-inference-fleet-active') + ON CONFLICT (name) DO NOTHING`, + ); + await prepareManagedReleaseCapabilities(pool, firstRelease, true); + const second = await prepareManagedReleaseCapabilities( + pool, + secondRelease, + true, + ); + expect(second.capabilities).toEqual([ "hosted-inference-fleet-active", + "publication-lifecycle-fleet-active", ]); + const pending = await pool.query<{ name: string }>( + `SELECT name FROM deployment_capabilities + WHERE name LIKE 'managed-release-preparation:%:root'`, + ); + expect(pending.rows.map((row) => row.name)).toEqual([ + `managed-release-preparation:${secondRelease}:${second.generation}:root`, + ]); + expect( + await restoreManagedReleasePreparation( + pool, + secondRelease, + second.generation, + ), + ).toBe(true); await activatePublicationLifecycleRelease(pool); }); diff --git a/tests/release-database-url.test.ts b/tests/release-database-url.test.ts index cac2960a..5730b900 100644 --- a/tests/release-database-url.test.ts +++ b/tests/release-database-url.test.ts @@ -148,6 +148,7 @@ describe("release database connection", () => { }; const snapshot = { releaseSha: "a".repeat(40), + generation: "00000000-0000-4000-8000-000000000001", publicationLifecycleReady: true, capabilities: [ "publication-lifecycle-fleet-active", @@ -188,6 +189,7 @@ describe("release database connection", () => { }; const snapshot = { releaseSha: "b".repeat(40), + generation: "00000000-0000-4000-8000-000000000002", publicationLifecycleReady: true, capabilities: ["publication-lifecycle-fleet-active"], }; @@ -235,6 +237,7 @@ describe("release database connection", () => { test("leaves durable compensation pending when child termination is unobservable", async () => { const snapshot = { releaseSha: "c".repeat(40), + generation: "00000000-0000-4000-8000-000000000003", publicationLifecycleReady: true, capabilities: ["publication-lifecycle-fleet-active"], }; @@ -291,6 +294,10 @@ describe("release database connection", () => { expect(productionMonitorWorkflow).toContain( "Postil release recovery failed", ); + expect(productionMonitorWorkflow).toContain("postil-release-recovery"); + expect(productionMonitorWorkflow).toContain( + "bun scripts/run-release-migrations.ts --verify-clear", + ); expect(deactivationScript).toContain("resolveDirectDatabaseUrl"); expect(deactivationScript).toContain("publication_lifecycle_required_at"); expect(deactivationScript.indexOf("process.env.DATABASE_URL =")).toBeLessThan( From 9a37d376f2d7bff7e2a552a02e73db5b74a2a360 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 02:32:28 +0000 Subject: [PATCH 23/34] Retain release recovery ownership --- .github/workflows/deploy.yml | 7 +++---- .github/workflows/production-monitor.yml | 21 ++++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e72f71be..c109811d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -362,10 +362,9 @@ jobs: target_seen=1 fi done < <(jq -r '.[] | select( - .state == "started" and - (.config.metadata.fly_process_group == "web" or - .config.metadata.fly_process_group == "worker" or - .config.metadata.fly_process_group == "monitor") + .config.metadata.fly_process_group == "web" or + .config.metadata.fly_process_group == "worker" or + .config.metadata.fly_process_group == "monitor" ) | .id' <<<"${machines}") if [[ "${#releases[@]}" -ne "${managed_count}" ]]; then echo "Managed fleet evidence is incomplete; release capabilities remain dark." diff --git a/.github/workflows/production-monitor.yml b/.github/workflows/production-monitor.yml index 613722e7..9d1c9ea9 100644 --- a/.github/workflows/production-monitor.yml +++ b/.github/workflows/production-monitor.yml @@ -72,11 +72,14 @@ jobs: POSTIL_RELEASE_SHA: ${{ github.event.workflow_run.head_sha }} run: | set -euo pipefail - latest_deploy_run_id=$(gh api \ + IFS=$'\t' read -r latest_deploy_run_id recovery_target_sha < <(gh api \ "repos/${GITHUB_REPOSITORY}/actions/workflows/deploy.yml/runs?per_page=1" \ - --jq '.workflow_runs[0].id') + --jq '.workflow_runs[0] | [.id, .head_sha] | @tsv') if [[ "${latest_deploy_run_id}" != "${FAILED_DEPLOY_RUN_ID}" ]]; then - echo "A newer deployment owns release recovery." + echo "Taking recovery ownership for the latest deployment run." + fi + if bun scripts/run-release-migrations.ts --verify-clear >/dev/null 2>&1; then + echo "A newer deployment already cleared release preparation." exit 0 fi machines=$(flyctl machine list --app postil-web --json) @@ -106,14 +109,13 @@ jobs: exit 1 fi releases+=("${release}") - if [[ "${release}" == "${POSTIL_RELEASE_SHA}" ]]; then + if [[ "${release}" == "${recovery_target_sha}" ]]; then target_seen=1 fi done < <(jq -r '.[] | select( - .state == "started" and - (.config.metadata.fly_process_group == "web" or - .config.metadata.fly_process_group == "worker" or - .config.metadata.fly_process_group == "monitor") + .config.metadata.fly_process_group == "web" or + .config.metadata.fly_process_group == "worker" or + .config.metadata.fly_process_group == "monitor" ) | .id' <<<"${machines}") unique_release_count=$(printf '%s\n' "${releases[@]}" | sort -u | wc -l) if [[ "${#releases[@]}" -ne "${managed_count}" || "${unique_release_count}" -ne 1 ]]; then @@ -124,7 +126,8 @@ jobs: echo "The target release reached the managed fleet; capabilities remain dark." exit 1 fi - bun scripts/run-release-migrations.ts --compensate + POSTIL_RELEASE_SHA="${recovery_target_sha}" \ + bun scripts/run-release-migrations.ts --compensate smoke: name: Smoke check production From 7eb06ca73d621d20f7bb5261c5ed6a7112437cd6 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 02:40:38 +0000 Subject: [PATCH 24/34] Keep lifecycle drains continuously queued --- ...ication_lifecycle_nonblocking_triggers.sql | 3 + src/lib/publication-lifecycle-lock.ts | 5 ++ src/lib/release-job-rollout.ts | 72 +++++++++---------- tests/private-worker-gates.test.ts | 5 +- 4 files changed, 43 insertions(+), 42 deletions(-) diff --git a/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql b/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql index 82369d67..a08a3c36 100644 --- a/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql +++ b/drizzle/0059_publication_lifecycle_nonblocking_triggers.sql @@ -1,3 +1,6 @@ +-- Release preparation darkens publication and drains operations tracked by the +-- v1 durable job and lease protocol before these triggers use the v2 key. The +-- distinct key prevents obsolete session locks from blocking this protocol. CREATE OR REPLACE FUNCTION "postil_require_publication_lifecycle"() RETURNS trigger LANGUAGE plpgsql diff --git a/src/lib/publication-lifecycle-lock.ts b/src/lib/publication-lifecycle-lock.ts index 59a39e46..ee876a61 100644 --- a/src/lib/publication-lifecycle-lock.ts +++ b/src/lib/publication-lifecycle-lock.ts @@ -2,6 +2,11 @@ import { sql } from "drizzle-orm"; import type { Database } from "@/lib/db"; +/** + * Release preparation drains operations tracked by the v1 durable job and + * lease protocol before migration 0059 activates this transaction-scoped key. + * A distinct key prevents obsolete session locks from blocking the protocol. + */ export const PUBLICATION_LIFECYCLE_LOCK = "postil:publication-lifecycle-release-v2"; diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index f40f1c39..170402aa 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -45,7 +45,6 @@ function databaseClientError(error: unknown, fallback: string): Error { async function lockPublicationLifecycleExclusive( client: PoolClient, ): Promise { - const deadline = Date.now() + PUBLICATION_LIFECYCLE_LOCK_TIMEOUT_MS; const configuredLockTimeout = await client.query<{ lock_timeout: string }>( "SHOW lock_timeout", ); @@ -55,50 +54,43 @@ async function lockPublicationLifecycleExclusive( "publication lifecycle lock timeout configuration is unavailable", ); } - while (true) { - const acquired = await client.query<{ acquired: boolean }>( - "SELECT pg_try_advisory_xact_lock(hashtextextended($1, 0)) AS acquired", + await client.query("SAVEPOINT publication_lifecycle_lock_attempt"); + try { + // Keep one exclusive request continuously queued. Trigger try-locks then + // defer new producers without a polling gap that could starve the drain. + await client.query("SELECT set_config('lock_timeout', $1, true)", [ + `${PUBLICATION_LIFECYCLE_LOCK_TIMEOUT_MS}ms`, + ]); + await client.query( + "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [PUBLICATION_LIFECYCLE_LOCK], ); - if (acquired.rows[0]?.acquired === true) return; - - await client.query("SAVEPOINT publication_lifecycle_lock_attempt"); + await client.query("SELECT set_config('lock_timeout', $1, true)", [ + lockTimeout, + ]); + await client.query("RELEASE SAVEPOINT publication_lifecycle_lock_attempt"); + } catch (error) { try { - // A bounded blocking request enters PostgreSQL's lock queue. Trigger - // try-locks then defer new producers instead of extending this drain. - await client.query("SET LOCAL lock_timeout = '250ms'"); - await client.query( - "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", - [PUBLICATION_LIFECYCLE_LOCK], - ); - await client.query("SELECT set_config('lock_timeout', $1, true)", [ - lockTimeout, - ]); + await client.query("ROLLBACK TO SAVEPOINT publication_lifecycle_lock_attempt"); await client.query("RELEASE SAVEPOINT publication_lifecycle_lock_attempt"); - return; - } catch (error) { - try { - await client.query("ROLLBACK TO SAVEPOINT publication_lifecycle_lock_attempt"); - await client.query("RELEASE SAVEPOINT publication_lifecycle_lock_attempt"); - } catch (cleanupError) { - throw new AggregateError( - [ - databaseClientError(error, "publication lifecycle lock attempt failed"), - databaseClientError( - cleanupError, - "publication lifecycle lock savepoint cleanup failed", - ), - ], - "publication lifecycle lock attempt and savepoint cleanup failed", - ); - } - if ((error as { code?: string }).code !== "55P03") throw error; - if (Date.now() >= deadline) { - throw new Error( - "publication lifecycle lock did not quiesce within 30 seconds", - ); - } + } catch (cleanupError) { + throw new AggregateError( + [ + databaseClientError(error, "publication lifecycle lock attempt failed"), + databaseClientError( + cleanupError, + "publication lifecycle lock savepoint cleanup failed", + ), + ], + "publication lifecycle lock attempt and savepoint cleanup failed", + ); } + if ((error as { code?: string }).code === "55P03") { + throw new Error( + "publication lifecycle lock did not quiesce within 30 seconds", + ); + } + throw error; } } diff --git a/tests/private-worker-gates.test.ts b/tests/private-worker-gates.test.ts index 58938e13..43cc145d 100644 --- a/tests/private-worker-gates.test.ts +++ b/tests/private-worker-gates.test.ts @@ -224,8 +224,9 @@ describe("private repository worker defense in depth", () => { expect(shared).not.toContain("pg_advisory_lock_shared"); expect(shared).not.toContain("pg_advisory_unlock_shared"); expect(activation).toContain("lockPublicationLifecycleExclusive(client)"); - expect(exclusiveLock).toContain("pg_try_advisory_xact_lock"); - expect(exclusiveLock).toContain("lock_timeout = '250ms'"); + expect(exclusiveLock).toContain("pg_advisory_xact_lock"); + expect(exclusiveLock).not.toContain("pg_try_advisory_xact_lock"); + expect(exclusiveLock).toContain("PUBLICATION_LIFECYCLE_LOCK_TIMEOUT_MS"); expect(exclusiveLock).toContain("set_config('lock_timeout', $1, true)"); expect(exclusiveLock).toContain("ROLLBACK TO SAVEPOINT"); expect(exclusiveLock).not.toContain("pg_terminate_backend"); From bed4c0f0d159735e4307ef3d6cc9f43b2aa82b9b Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 02:53:58 +0000 Subject: [PATCH 25/34] Clarify failed release recovery boundaries --- .github/workflows/deploy.yml | 4 ++++ .github/workflows/production-monitor.yml | 6 ++++-- tests/release-database-url.test.ts | 3 +++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c109811d..412d77cc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -372,10 +372,14 @@ jobs: fi unique_release_count=$(printf '%s\n' "${releases[@]}" | sort -u | wc -l) if [[ "${unique_release_count}" -ne 1 ]]; then + # Re-enabling publication while any target machine remains would + # let an unverified mixed fleet publish against restored state. echo "The managed fleet is mixed; release capabilities remain dark." exit 1 fi if [[ "${target_seen}" -ne 0 ]]; then + # The target code reached every managed machine. A failed deploy + # needs activation or rollback proof, not prior-state restoration. echo "The target release reached the managed fleet; release capabilities remain dark." exit 1 fi diff --git a/.github/workflows/production-monitor.yml b/.github/workflows/production-monitor.yml index 9d1c9ea9..4adfd8b2 100644 --- a/.github/workflows/production-monitor.yml +++ b/.github/workflows/production-monitor.yml @@ -78,8 +78,10 @@ jobs: if [[ "${latest_deploy_run_id}" != "${FAILED_DEPLOY_RUN_ID}" ]]; then echo "Taking recovery ownership for the latest deployment run." fi + # Durable database state is authoritative. No journal plus both + # active fleet capabilities means no release preparation remains. if bun scripts/run-release-migrations.ts --verify-clear >/dev/null 2>&1; then - echo "A newer deployment already cleared release preparation." + echo "Release preparation is already active and clear." exit 0 fi machines=$(flyctl machine list --app postil-web --json) @@ -524,7 +526,7 @@ jobs: with: event-type: ALERT summary: ${{ needs.release-recovery.result == 'failure' && 'Postil release recovery failed' || needs.smoke.result == 'failure' && 'Postil production monitor failed' || 'Postil production monitor test alert' }} - alert-key: ${{ needs.release-recovery.result == 'failure' && 'postil-release-recovery' || inputs.test_alert == true && 'postil-production-monitor-test' || 'postil-production-monitor' }} + alert-key: ${{ needs.release-recovery.result == 'failure' && 'postil-release-recovery' || needs.smoke.result == 'failure' && 'postil-production-monitor' || 'postil-production-monitor-test' }} details: >- ${{ needs.release-recovery.result == 'failure' && 'Failed deployment left release recovery unresolved. Run log:' diff --git a/tests/release-database-url.test.ts b/tests/release-database-url.test.ts index 5730b900..40708e30 100644 --- a/tests/release-database-url.test.ts +++ b/tests/release-database-url.test.ts @@ -295,6 +295,9 @@ describe("release database connection", () => { "Postil release recovery failed", ); expect(productionMonitorWorkflow).toContain("postil-release-recovery"); + expect(productionMonitorWorkflow).toContain( + "needs.release-recovery.result == 'failure' && 'postil-release-recovery' || needs.smoke.result == 'failure' && 'postil-production-monitor' || 'postil-production-monitor-test'", + ); expect(productionMonitorWorkflow).toContain( "bun scripts/run-release-migrations.ts --verify-clear", ); From 6c15332f169ac68fd04ed78c7d20a21561bc59d1 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 03:15:55 +0000 Subject: [PATCH 26/34] Make release recovery atomic and queued --- .github/workflows/deploy.yml | 1 + .github/workflows/production-monitor.yml | 17 +- src/lib/release-job-rollout.ts | 331 +++++++++++++------- tests/private-worker-gates.test.ts | 13 + tests/publication-receipt-migration.test.ts | 166 +++++++++- tests/release-database-url.test.ts | 42 ++- 6 files changed, 449 insertions(+), 121 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 412d77cc..8f3a38fe 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,6 +13,7 @@ on: concurrency: group: fly-deploy + queue: max cancel-in-progress: false permissions: diff --git a/.github/workflows/production-monitor.yml b/.github/workflows/production-monitor.yml index 4adfd8b2..488ab5dd 100644 --- a/.github/workflows/production-monitor.yml +++ b/.github/workflows/production-monitor.yml @@ -23,7 +23,11 @@ permissions: contents: read concurrency: - group: production-monitor + # Deployment-completion monitors keep independent workflow owners, while + # scheduled checks serialize together. The recovery job then joins the same + # bounded FIFO queue as deploys, so no pending recovery is replaced. + group: ${{ github.event_name == 'workflow_run' && format('production-monitor-deploy-{0}', github.event.workflow_run.id) || 'production-monitor' }} + queue: max cancel-in-progress: false jobs: @@ -32,6 +36,7 @@ jobs: if: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.conclusion != 'success' }} concurrency: group: fly-deploy + queue: max cancel-in-progress: false permissions: actions: read @@ -504,7 +509,7 @@ jobs: notify: name: Raise external alert needs: [smoke, release-recovery] - if: ${{ always() && (needs.smoke.result == 'failure' || needs.release-recovery.result == 'failure' || inputs.test_alert == true) }} + if: ${{ always() && (needs.smoke.result == 'failure' || needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled' || inputs.test_alert == true) }} permissions: contents: read id-token: write @@ -525,10 +530,10 @@ jobs: uses: ./.github/actions/ilert-event with: event-type: ALERT - summary: ${{ needs.release-recovery.result == 'failure' && 'Postil release recovery failed' || needs.smoke.result == 'failure' && 'Postil production monitor failed' || 'Postil production monitor test alert' }} - alert-key: ${{ needs.release-recovery.result == 'failure' && 'postil-release-recovery' || needs.smoke.result == 'failure' && 'postil-production-monitor' || 'postil-production-monitor-test' }} + summary: ${{ (needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') && 'Postil release recovery failed' || needs.smoke.result == 'failure' && 'Postil production monitor failed' || 'Postil production monitor test alert' }} + alert-key: ${{ (needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') && 'postil-release-recovery' || needs.smoke.result == 'failure' && 'postil-production-monitor' || 'postil-production-monitor-test' }} details: >- - ${{ needs.release-recovery.result == 'failure' + ${{ (needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') && 'Failed deployment left release recovery unresolved. Run log:' || needs.smoke.result == 'failure' && 'Production checks failed. Run log:' @@ -537,7 +542,7 @@ jobs: # A routine monitor failure records an alerting gap without masking # the original signal. Recovery failure and test events require # delivery because they validate the fail-safe notification path. - require-delivery: ${{ inputs.test_alert == true || needs.release-recovery.result == 'failure' }} + require-delivery: ${{ inputs.test_alert == true || needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled' }} resolve-release-recovery: name: Resolve release recovery alert diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index 170402aa..4830d80c 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -185,8 +185,8 @@ export async function deactivatePublicationLifecycleRelease( await client.query("BEGIN"); transactionOpen = true; - await lockPublicationLifecycleExclusive(client); await waitForLegacyPublicationLifecycleOperations(client); + await lockPublicationLifecycleExclusive(client); const fenced = await darkenPublicationLifecycle(client); await client.query("COMMIT"); transactionOpen = false; @@ -195,6 +195,16 @@ export async function deactivatePublicationLifecycleRelease( parked: initial.parked + fenced.parked, }; } catch (error) { + const primaryError = databaseClientError( + error, + "publication lifecycle deactivation failed", + ); + if (!transactionOpen) { + // A failed BEGIN leaves the backend state uncertain. Do not return that + // client to the pool where a later operation could inherit the failure. + releaseError = primaryError; + throw primaryError; + } if (transactionOpen) { try { await client.query("ROLLBACK"); @@ -205,17 +215,14 @@ export async function deactivatePublicationLifecycleRelease( ); throw new AggregateError( [ - databaseClientError( - error, - "publication lifecycle deactivation failed", - ), + primaryError, releaseError, ], "publication lifecycle deactivation and rollback failed", ); } } - throw error; + throw primaryError; } finally { client.release(releaseError); } @@ -732,7 +739,7 @@ function managedReleasePreparationSnapshot( }; } -async function captureManagedReleaseCapabilities( +async function captureAndDarkenManagedReleaseCapabilities( pool: Pool, releaseSha: string, publicationLifecycleReady: boolean, @@ -741,15 +748,20 @@ async function captureManagedReleaseCapabilities( const generation = randomUUID(); const journal = managedReleasePreparationNames(releaseSha, generation); const client = await pool.connect(); + let releaseError: Error | undefined; + let transactionOpen = false; try { await client.query("BEGIN"); - if (publicationLifecycleReady) { - await lockPublicationLifecycleExclusive(client); - } + transactionOpen = true; + await lockPublicationLifecycleExclusive(client); await client.query( "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [HOSTED_INFERENCE_LOCK], ); + // Adopt abandoned generations under the same locks and transaction that + // immediately captures and darkens the replacement. Their desired state + // is never committed as active while the fleet may still be mixed. + await restoreAllManagedReleasePreparationsOnClient(client); const existing = await client.query<{ name: string }>( "SELECT name FROM deployment_capabilities WHERE name = ANY($1::text[]) ORDER BY name", [names], @@ -782,13 +794,50 @@ async function captureManagedReleaseCapabilities( ON CONFLICT (name) DO UPDATE SET activated_at = now()`, [journalNames], ); + if (publicationLifecycleReady) { + await darkenPublicationLifecycle(client); + } + await client.query( + "DELETE FROM deployment_capabilities WHERE name = $1", + [hostedInferenceCapability(releaseSha)], + ); + await client.query( + "DELETE FROM deployment_capabilities WHERE name = $1", + [HOSTED_INFERENCE_FLEET_ACTIVE_CAPABILITY], + ); + await client.query( + `INSERT INTO deployment_capabilities (name) + VALUES ($1) + ON CONFLICT (name) DO NOTHING`, + [hostedInferenceDarkCapability(releaseSha)], + ); await client.query("COMMIT"); + transactionOpen = false; return snapshot; } catch (error) { - await client.query("ROLLBACK").catch(() => undefined); - throw error; + const primaryError = databaseClientError( + error, + "managed release capability capture failed", + ); + if (!transactionOpen) { + releaseError = primaryError; + throw primaryError; + } + try { + await client.query("ROLLBACK"); + } catch (rollbackError) { + releaseError = databaseClientError( + rollbackError, + "managed release capability capture rollback failed", + ); + throw new AggregateError( + [primaryError, releaseError], + "managed release capability capture and rollback failed", + ); + } + throw primaryError; } finally { - client.release(); + client.release(releaseError); } } @@ -799,8 +848,7 @@ export async function prepareManagedReleaseCapabilities( publicationLifecycleReady: boolean, ): Promise { const normalizedRelease = normalizedReleaseSha(releaseSha); - await restoreAllManagedReleasePreparations(pool); - const snapshot = await captureManagedReleaseCapabilities( + const snapshot = await captureAndDarkenManagedReleaseCapabilities( pool, normalizedRelease, publicationLifecycleReady, @@ -809,7 +857,6 @@ export async function prepareManagedReleaseCapabilities( if (publicationLifecycleReady) { await deactivatePublicationLifecycleRelease(pool); } - await deactivateHostedInferenceRelease(pool, normalizedRelease); return snapshot; } catch (error) { try { @@ -842,88 +889,33 @@ async function restoreManagedReleaseCapabilitiesInternal( pool: Pool, snapshot: ManagedReleaseCapabilitySnapshot, ): Promise { - const names = managedReleaseCapabilityNames(snapshot.releaseSha); - const journal = managedReleasePreparationNames( - snapshot.releaseSha, - snapshot.generation, - ); - const journalNames = Object.values(journal); const client = await pool.connect(); let releaseError: Error | undefined; + let transactionOpen = false; try { await client.query("BEGIN"); + transactionOpen = true; await lockPublicationLifecycleExclusive(client); await client.query( "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [HOSTED_INFERENCE_LOCK], ); - const durable = await client.query<{ name: string }>( - "SELECT name FROM deployment_capabilities WHERE name = ANY($1::text[])", - [journalNames], - ); - const effectiveSnapshot = managedReleasePreparationSnapshot( - snapshot.releaseSha, - snapshot.generation, - durable.rows.map((row) => row.name), - ); - if (!effectiveSnapshot) { - await client.query("COMMIT"); - return false; - } - const expected = new Set(names); - if ( - effectiveSnapshot.capabilities.some((name) => !expected.has(name)) || - new Set(effectiveSnapshot.capabilities).size !== - effectiveSnapshot.capabilities.length - ) { - throw new Error("managed release capability snapshot is invalid"); - } - const publicationWasActive = effectiveSnapshot.capabilities.includes( - PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY, - ); - await client.query( - "DELETE FROM deployment_capabilities WHERE name = ANY($1::text[])", - [names], - ); - if (effectiveSnapshot.capabilities.length > 0) { - await client.query( - `INSERT INTO deployment_capabilities (name) - SELECT unnest($1::text[])`, - [effectiveSnapshot.capabilities], - ); - } - if (effectiveSnapshot.publicationLifecycleReady && publicationWasActive) { - await client.query( - `UPDATE jobs - SET run_after = now(), payload = payload - $1 - WHERE kind = 'gate-state-sync' - AND status = 'queued' - AND payload ? $1`, - [PUBLICATION_LIFECYCLE_DARK_PAYLOAD_KEY], - ); - } - if ( - effectiveSnapshot.capabilities.includes( - HOSTED_INFERENCE_FLEET_ACTIVE_CAPABILITY, - ) - ) { - await client.query( - `UPDATE jobs - SET run_after = now(), payload = payload - 'releaseDarkSha' - WHERE kind IN ('review', $1) - AND status = 'queued' - AND run_after = 'infinity'::timestamptz - AND payload ? 'releaseDarkSha'`, - [HOSTED_PROVIDER_KEY_LIFECYCLE_JOB_KIND], - ); - } - await client.query( - "DELETE FROM deployment_capabilities WHERE name = ANY($1::text[])", - [journalNames], + const restored = await restoreManagedReleaseCapabilitiesOnClient( + client, + snapshot, ); await client.query("COMMIT"); - return true; + transactionOpen = false; + return restored; } catch (error) { + const primaryError = databaseClientError( + error, + "managed release capability compensation failed", + ); + if (!transactionOpen) { + releaseError = primaryError; + throw primaryError; + } try { await client.query("ROLLBACK"); } catch (rollbackError) { @@ -932,22 +924,120 @@ async function restoreManagedReleaseCapabilitiesInternal( "managed release capability compensation rollback failed", ); throw new AggregateError( - [ - databaseClientError( - error, - "managed release capability compensation failed", - ), - releaseError, - ], + [primaryError, releaseError], "managed release capability compensation and rollback failed", ); } - throw error; + throw primaryError; } finally { client.release(releaseError); } } +async function restoreManagedReleaseCapabilitiesOnClient( + client: PoolClient, + snapshot: ManagedReleaseCapabilitySnapshot, +): Promise { + const names = managedReleaseCapabilityNames(snapshot.releaseSha); + const journal = managedReleasePreparationNames( + snapshot.releaseSha, + snapshot.generation, + ); + const journalNames = Object.values(journal); + const durable = await client.query<{ name: string }>( + "SELECT name FROM deployment_capabilities WHERE name = ANY($1::text[])", + [journalNames], + ); + const effectiveSnapshot = managedReleasePreparationSnapshot( + snapshot.releaseSha, + snapshot.generation, + durable.rows.map((row) => row.name), + ); + if (!effectiveSnapshot) return false; + const expected = new Set(names); + if ( + effectiveSnapshot.capabilities.some((name) => !expected.has(name)) || + new Set(effectiveSnapshot.capabilities).size !== + effectiveSnapshot.capabilities.length + ) { + throw new Error("managed release capability snapshot is invalid"); + } + const publicationWasActive = effectiveSnapshot.capabilities.includes( + PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY, + ); + await client.query( + "DELETE FROM deployment_capabilities WHERE name = ANY($1::text[])", + [names], + ); + if (effectiveSnapshot.capabilities.length > 0) { + await client.query( + `INSERT INTO deployment_capabilities (name) + SELECT unnest($1::text[])`, + [effectiveSnapshot.capabilities], + ); + } + if (effectiveSnapshot.publicationLifecycleReady && publicationWasActive) { + await client.query( + `UPDATE jobs + SET run_after = now(), payload = payload - $1 + WHERE kind = 'gate-state-sync' + AND status = 'queued' + AND payload ? $1`, + [PUBLICATION_LIFECYCLE_DARK_PAYLOAD_KEY], + ); + } + if ( + effectiveSnapshot.capabilities.includes( + HOSTED_INFERENCE_FLEET_ACTIVE_CAPABILITY, + ) + ) { + await client.query( + `UPDATE jobs + SET run_after = now(), payload = payload - 'releaseDarkSha' + WHERE kind IN ('review', $1) + AND status = 'queued' + AND run_after = 'infinity'::timestamptz + AND payload ? 'releaseDarkSha'`, + [HOSTED_PROVIDER_KEY_LIFECYCLE_JOB_KIND], + ); + } + await client.query( + "DELETE FROM deployment_capabilities WHERE name = ANY($1::text[])", + [journalNames], + ); + return true; +} + +async function restoreAllManagedReleasePreparationsOnClient( + client: PoolClient, +): Promise { + const roots = await client.query<{ name: string }>( + `SELECT name + FROM deployment_capabilities + WHERE name LIKE $1 + AND name LIKE '%:root' + ORDER BY activated_at DESC, name DESC`, + [`${MANAGED_RELEASE_PREPARATION_PREFIX}%`], + ); + let restored = 0; + for (const row of roots.rows) { + const match = row.name.match( + /^managed-release-preparation:([0-9a-f]{7,40}):([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}):root$/, + ); + if (!match) continue; + const candidate: ManagedReleaseCapabilitySnapshot = { + releaseSha: match[1]!, + generation: match[2]!, + publicationLifecycleReady: false, + capabilities: [], + }; + if (await restoreManagedReleaseCapabilitiesOnClient(client, candidate)) { + restored += 1; + } + } + return restored; +} + export async function restoreManagedReleasePreparation( pool: Pool, releaseSha: string, @@ -978,25 +1068,46 @@ export async function restoreManagedReleasePreparation( export async function restoreAllManagedReleasePreparations( pool: Pool, ): Promise { - const roots = await pool.query<{ name: string }>( - `SELECT name - FROM deployment_capabilities - WHERE name LIKE $1 - AND name LIKE '%:root' - ORDER BY activated_at DESC, name DESC`, - [`${MANAGED_RELEASE_PREPARATION_PREFIX}%`], - ); - let restored = 0; - for (const row of roots.rows) { - const match = row.name.match( - /^managed-release-preparation:([0-9a-f]{7,40}):([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}):root$/, + const client = await pool.connect(); + let releaseError: Error | undefined; + let transactionOpen = false; + try { + await client.query("BEGIN"); + transactionOpen = true; + await lockPublicationLifecycleExclusive(client); + await client.query( + "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", + [HOSTED_INFERENCE_LOCK], ); - if (!match) continue; - if (await restoreManagedReleasePreparation(pool, match[1]!, match[2]!)) { - restored += 1; + const restored = await restoreAllManagedReleasePreparationsOnClient(client); + await client.query("COMMIT"); + transactionOpen = false; + return restored; + } catch (error) { + const primaryError = databaseClientError( + error, + "managed release preparation recovery failed", + ); + if (!transactionOpen) { + releaseError = primaryError; + throw primaryError; } + try { + await client.query("ROLLBACK"); + } catch (rollbackError) { + releaseError = databaseClientError( + rollbackError, + "managed release preparation recovery rollback failed", + ); + throw new AggregateError( + [primaryError, releaseError], + "managed release preparation recovery and rollback failed", + ); + } + throw primaryError; + } finally { + client.release(releaseError); } - return restored; } /** Atomically park a claimed hosted review until a verified managed release activates. */ diff --git a/tests/private-worker-gates.test.ts b/tests/private-worker-gates.test.ts index 43cc145d..06c36fd7 100644 --- a/tests/private-worker-gates.test.ts +++ b/tests/private-worker-gates.test.ts @@ -204,6 +204,13 @@ describe("private repository worker defense in depth", () => { "export class PublicationLifecycleReleaseDarkError", exclusiveLockStart, ); + const deactivationStart = rollout.indexOf( + "export async function deactivatePublicationLifecycleRelease", + ); + const deactivationEnd = rollout.indexOf( + "async function darkenPublicationLifecycle", + deactivationStart, + ); const decisionStart = decisions.indexOf( "export async function withReviewDecisionScopeLock", ); @@ -215,6 +222,7 @@ describe("private repository worker defense in depth", () => { const shared = rollout.slice(sharedStart, sharedEnd); const activation = rollout.slice(activationStart, activationEnd); const exclusiveLock = rollout.slice(exclusiveLockStart, exclusiveLockEnd); + const deactivation = rollout.slice(deactivationStart, deactivationEnd); const decision = decisions.slice(decisionStart, decisionEnd); expect(lifecycleLock).toContain("pg_advisory_xact_lock_shared"); expect(shared).toContain("withPinnedDatabaseTransaction"); @@ -233,6 +241,11 @@ describe("private repository worker defense in depth", () => { expect(rollout).toContain( "waitForLegacyPublicationLifecycleOperations(client)", ); + expect( + deactivation.indexOf("waitForLegacyPublicationLifecycleOperations(client)"), + ).toBeLessThan( + deactivation.indexOf("lockPublicationLifecycleExclusive(client)"), + ); expect(rollout).toContain("kind = 'gate-state-sync'"); expect(rollout).toContain("status = 'running'"); expect(exclusiveLock).not.toContain("pg_stat_activity"); diff --git a/tests/publication-receipt-migration.test.ts b/tests/publication-receipt-migration.test.ts index 56888967..2790e53a 100644 --- a/tests/publication-receipt-migration.test.ts +++ b/tests/publication-receipt-migration.test.ts @@ -124,6 +124,27 @@ function envelope(input: { }; } +describe("publication lifecycle database client safety", () => { + test("discards a client when transaction start fails", async () => { + const beginError = new Error("transaction start failed"); + const releasedWith: Array = []; + const client = { + query: async () => { + throw beginError; + }, + release: (error?: Error) => releasedWith.push(error), + }; + const pool = { + connect: async () => client, + } as unknown as Pool; + + await expect(deactivatePublicationLifecycleRelease(pool)).rejects.toBe( + beginError, + ); + expect(releasedWith).toEqual([beginError]); + }); +}); + describeDb("publication receipt migration and lifecycle", () => { const pool = new Pool({ connectionString: TEST_URL, max: 2 }); const db = drizzle(pool, { schema }); @@ -833,6 +854,61 @@ describeDb("publication receipt migration and lifecycle", () => { } }); + test("legacy drain does not hold the lifecycle-v2 exclusive lock", async () => { + await pool.query( + `INSERT INTO deployment_capabilities (name) + VALUES ('publication-lifecycle-fleet-active') + ON CONFLICT (name) DO NOTHING`, + ); + const legacyJob = await pool.query<{ id: string }>( + `INSERT INTO jobs (kind, payload, status, locked_at, locked_by) + VALUES ( + 'gate-state-sync', + '{"reviewId":1,"reviewPublicId":"legacy-drain-order"}'::jsonb, + 'running', now(), 'legacy-worker' + ) + RETURNING id`, + ); + const deactivation = deactivatePublicationLifecycleRelease(pool); + const observer = await pool.connect(); + let darkVisible = false; + let sharedLockAcquired = false; + try { + for (let attempt = 0; attempt < 20; attempt += 1) { + const active = await observer.query<{ active: boolean }>( + `SELECT EXISTS ( + SELECT 1 FROM deployment_capabilities + WHERE name = 'publication-lifecycle-fleet-active' + ) AS active`, + ); + darkVisible = active.rows[0]?.active === false; + if (darkVisible) break; + await Bun.sleep(25); + } + await observer.query("BEGIN"); + const probe = await observer.query<{ acquired: boolean }>( + `SELECT pg_try_advisory_xact_lock_shared( + hashtextextended($1, 0) + ) AS acquired`, + ["postil:publication-lifecycle-release-v2"], + ); + sharedLockAcquired = probe.rows[0]?.acquired === true; + } finally { + await observer.query("ROLLBACK").catch(() => undefined); + await observer.query( + `UPDATE jobs + SET status = 'done', locked_at = NULL, locked_by = NULL + WHERE id = $1`, + [legacyJob.rows[0]!.id], + ); + observer.release(); + } + await expect(deactivation).resolves.toMatchObject({ deactivated: true }); + expect(darkVisible).toBe(true); + expect(sharedLockAcquired).toBe(true); + await activatePublicationLifecycleRelease(pool); + }); + test("deactivation drains a durable legacy publication operation without trusting backend state", async () => { const legacyPool = new Pool({ connectionString: TEST_URL, max: 1 }); const holder = await legacyPool.connect(); @@ -1099,12 +1175,89 @@ describeDb("publication receipt migration and lifecycle", () => { ('hosted-inference-fleet-active') ON CONFLICT (name) DO NOTHING`, ); - await prepareManagedReleaseCapabilities(pool, firstRelease, true); - const second = await prepareManagedReleaseCapabilities( + const gate = await pool.query<{ id: string }>( + `INSERT INTO jobs (kind, payload) + VALUES ('gate-state-sync', '{"reviewId":1,"reviewPublicId":"atomic-adoption"}'::jsonb) + RETURNING id`, + ); + const first = await prepareManagedReleaseCapabilities( pool, - secondRelease, + firstRelease, true, ); + await pool.query( + "DROP TRIGGER IF EXISTS test_pause_replacement_journal ON deployment_capabilities", + ); + await pool.query("DROP FUNCTION IF EXISTS test_pause_replacement_journal()"); + await pool.query(` + CREATE FUNCTION test_pause_replacement_journal() + RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.name LIKE 'managed-release-preparation:${secondRelease}:%:root' THEN + PERFORM pg_sleep(0.75); + END IF; + RETURN NEW; + END + $$ + `); + await pool.query(` + CREATE TRIGGER test_pause_replacement_journal + BEFORE INSERT ON deployment_capabilities + FOR EACH ROW EXECUTE FUNCTION test_pause_replacement_journal() + `); + let second!: Awaited>; + try { + const replacement = prepareManagedReleaseCapabilities( + pool, + secondRelease, + true, + ); + let barrierReached = false; + for (let attempt = 0; attempt < 40; attempt += 1) { + const barrier = await pool.query<{ waiting: boolean }>( + `SELECT EXISTS ( + SELECT 1 FROM pg_stat_activity + WHERE pid <> pg_backend_pid() + AND state = 'active' + AND wait_event = 'PgSleep' + AND query LIKE '%ON CONFLICT (name) DO UPDATE SET activated_at = now()%' + ) AS waiting`, + ); + barrierReached = barrier.rows[0]?.waiting === true; + if (barrierReached) break; + await Bun.sleep(25); + } + const visibleDuringAdoption = await pool.query<{ name: string }>( + `SELECT name FROM deployment_capabilities + WHERE name = ANY($1::text[]) + OR name LIKE 'managed-release-preparation:%:root' + ORDER BY name`, + [[ + "publication-lifecycle-fleet-active", + "hosted-inference-fleet-active", + ]], + ); + expect(barrierReached).toBe(true); + expect(visibleDuringAdoption.rows.map((row) => row.name)).toEqual([ + `managed-release-preparation:${firstRelease}:${first.generation}:root`, + ]); + const gateDuringAdoption = await pool.query<{ + parked: boolean; + dark: boolean; + }>( + `SELECT run_after = 'infinity'::timestamptz AS parked, + payload ? '_postilPublicationLifecycleDark' AS dark + FROM jobs WHERE id = $1`, + [gate.rows[0]!.id], + ); + expect(gateDuringAdoption.rows[0]).toEqual({ parked: true, dark: true }); + second = await replacement; + } finally { + await pool.query( + "DROP TRIGGER IF EXISTS test_pause_replacement_journal ON deployment_capabilities", + ); + await pool.query("DROP FUNCTION IF EXISTS test_pause_replacement_journal()"); + } expect(second.capabilities).toEqual([ "hosted-inference-fleet-active", "publication-lifecycle-fleet-active", @@ -1123,6 +1276,13 @@ describeDb("publication receipt migration and lifecycle", () => { second.generation, ), ).toBe(true); + const restoredGate = await pool.query<{ due: boolean; dark: boolean }>( + `SELECT run_after <= now() AS due, + payload ? '_postilPublicationLifecycleDark' AS dark + FROM jobs WHERE id = $1`, + [gate.rows[0]!.id], + ); + expect(restoredGate.rows[0]).toEqual({ due: true, dark: false }); await activatePublicationLifecycleRelease(pool); }); diff --git a/tests/release-database-url.test.ts b/tests/release-database-url.test.ts index 40708e30..40732792 100644 --- a/tests/release-database-url.test.ts +++ b/tests/release-database-url.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { parse } from "yaml"; import { resolveDirectDatabaseUrl } from "../scripts/resolve-direct-database-url"; import { @@ -268,6 +269,21 @@ describe("release database connection", () => { join(root, ".github", "workflows", "production-monitor.yml"), "utf8", ); + const deployWorkflowConfig = parse(deployWorkflow) as { + concurrency: { group: string; queue: string; "cancel-in-progress": boolean }; + }; + const productionMonitorConfig = parse(productionMonitorWorkflow) as { + concurrency: { group: string; queue: string; "cancel-in-progress": boolean }; + jobs: { + "release-recovery": { + concurrency: { + group: string; + queue: string; + "cancel-in-progress": boolean; + }; + }; + }; + }; const deactivationScript = await readFile( join(root, "scripts", "deactivate-hosted-inference.ts"), "utf8", @@ -286,7 +302,26 @@ describe("release database connection", () => { ); expect(deployWorkflow).toContain("bun scripts/run-release-migrations.ts --compensate"); expect(productionMonitorWorkflow).toContain('workflows: ["deploy"]'); - expect(productionMonitorWorkflow).toContain("group: fly-deploy"); + expect(deployWorkflowConfig.concurrency).toEqual({ + group: "fly-deploy", + queue: "max", + "cancel-in-progress": false, + }); + expect(productionMonitorConfig.concurrency.queue).toBe("max"); + expect(productionMonitorConfig.concurrency["cancel-in-progress"]).toBe(false); + expect(productionMonitorConfig.concurrency.group).toContain( + "production-monitor-deploy-{0}", + ); + expect(productionMonitorConfig.concurrency.group).toContain( + "github.event.workflow_run.id", + ); + expect( + productionMonitorConfig.jobs["release-recovery"].concurrency, + ).toEqual({ + group: "fly-deploy", + queue: "max", + "cancel-in-progress": false, + }); expect(productionMonitorWorkflow).toContain("latest_deploy_run_id"); expect(productionMonitorWorkflow).toContain( "needs: [smoke, release-recovery]", @@ -296,7 +331,10 @@ describe("release database connection", () => { ); expect(productionMonitorWorkflow).toContain("postil-release-recovery"); expect(productionMonitorWorkflow).toContain( - "needs.release-recovery.result == 'failure' && 'postil-release-recovery' || needs.smoke.result == 'failure' && 'postil-production-monitor' || 'postil-production-monitor-test'", + "needs.release-recovery.result == 'cancelled') && 'postil-release-recovery'", + ); + expect(productionMonitorWorkflow).toContain( + "needs.release-recovery.result == 'cancelled'", ); expect(productionMonitorWorkflow).toContain( "bun scripts/run-release-migrations.ts --verify-clear", From 730835263e0edd461891b8edb10c599d29413afc Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 03:21:26 +0000 Subject: [PATCH 27/34] Fence superseded release recovery owners --- .github/workflows/production-monitor.yml | 11 ++++++++--- tests/release-database-url.test.ts | 6 ++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/production-monitor.yml b/.github/workflows/production-monitor.yml index 488ab5dd..47e5e7b1 100644 --- a/.github/workflows/production-monitor.yml +++ b/.github/workflows/production-monitor.yml @@ -77,12 +77,17 @@ jobs: POSTIL_RELEASE_SHA: ${{ github.event.workflow_run.head_sha }} run: | set -euo pipefail - IFS=$'\t' read -r latest_deploy_run_id recovery_target_sha < <(gh api \ + latest_deploy_run_id=$(gh api \ "repos/${GITHUB_REPOSITORY}/actions/workflows/deploy.yml/runs?per_page=1" \ - --jq '.workflow_runs[0] | [.id, .head_sha] | @tsv') + --jq '.workflow_runs[0].id') if [[ "${latest_deploy_run_id}" != "${FAILED_DEPLOY_RUN_ID}" ]]; then - echo "Taking recovery ownership for the latest deployment run." + # queue:max preserves a separate recovery owner for every deploy + # completion. A superseded owner must not mutate the newer run's + # durable preparation or compensate against its fleet evidence. + echo "A newer deployment run owns release recovery." + exit 0 fi + recovery_target_sha="${POSTIL_RELEASE_SHA}" # Durable database state is authoritative. No journal plus both # active fleet capabilities means no release preparation remains. if bun scripts/run-release-migrations.ts --verify-clear >/dev/null 2>&1; then diff --git a/tests/release-database-url.test.ts b/tests/release-database-url.test.ts index 40732792..94347e43 100644 --- a/tests/release-database-url.test.ts +++ b/tests/release-database-url.test.ts @@ -323,6 +323,12 @@ describe("release database connection", () => { "cancel-in-progress": false, }); expect(productionMonitorWorkflow).toContain("latest_deploy_run_id"); + expect(productionMonitorWorkflow).toContain( + "A newer deployment run owns release recovery.", + ); + expect(productionMonitorWorkflow).toContain( + 'recovery_target_sha="${POSTIL_RELEASE_SHA}"', + ); expect(productionMonitorWorkflow).toContain( "needs: [smoke, release-recovery]", ); From f203c199f213bd4c3455f60a649f82b5f81de513 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 03:26:14 +0000 Subject: [PATCH 28/34] Discard uncertain activation clients --- src/lib/release-job-rollout.ts | 15 +++++++++-- tests/publication-receipt-migration.test.ts | 29 +++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index 4830d80c..ac13df5f 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -269,8 +269,10 @@ export async function activatePublicationLifecycleRelease( }> { const client = await pool.connect(); let releaseError: Error | undefined; + let transactionOpen = false; try { await client.query("BEGIN"); + transactionOpen = true; await lockPublicationLifecycleExclusive(client); const invalid = await client.query<{ count: string }>( `SELECT count(*)::text AS count @@ -379,6 +381,7 @@ export async function activatePublicationLifecycleRelease( [PUBLICATION_LIFECYCLE_DARK_PAYLOAD_KEY], ); await client.query("COMMIT"); + transactionOpen = false; return { activated: (activated.rowCount ?? 0) > 0, recoveriesQueued: recoveries.rowCount ?? 0, @@ -386,6 +389,14 @@ export async function activatePublicationLifecycleRelease( released: released.rowCount ?? 0, }; } catch (error) { + const primaryError = databaseClientError( + error, + "publication lifecycle activation failed", + ); + if (!transactionOpen) { + releaseError = primaryError; + throw primaryError; + } try { await client.query("ROLLBACK"); } catch (rollbackError) { @@ -394,11 +405,11 @@ export async function activatePublicationLifecycleRelease( "publication lifecycle activation rollback failed", ); throw new AggregateError( - [databaseClientError(error, "publication lifecycle activation failed"), releaseError], + [primaryError, releaseError], "publication lifecycle activation and rollback failed", ); } - throw error; + throw primaryError; } finally { client.release(releaseError); } diff --git a/tests/publication-receipt-migration.test.ts b/tests/publication-receipt-migration.test.ts index 2790e53a..fc3b291c 100644 --- a/tests/publication-receipt-migration.test.ts +++ b/tests/publication-receipt-migration.test.ts @@ -143,6 +143,35 @@ describe("publication lifecycle database client safety", () => { ); expect(releasedWith).toEqual([beginError]); }); + + test("discards an activation client when rollback fails", async () => { + const primaryError = new Error("activation query failed"); + const rollbackError = new Error("activation rollback failed"); + const releasedWith: Array = []; + const client = { + query: async (statement: string) => { + if (statement === "SHOW lock_timeout") { + return { rows: [{ lock_timeout: "0" }], rowCount: 1 }; + } + if (statement.includes("SELECT count(*)::text AS count")) { + throw primaryError; + } + if (statement === "ROLLBACK") throw rollbackError; + return { rows: [], rowCount: 0 }; + }, + release: (error?: Error) => releasedWith.push(error), + }; + const pool = { + connect: async () => client, + } as unknown as Pool; + + const result = activatePublicationLifecycleRelease(pool); + await expect(result).rejects.toBeInstanceOf(AggregateError); + await expect(result).rejects.toThrow( + "publication lifecycle activation and rollback failed", + ); + expect(releasedWith).toEqual([rollbackError]); + }); }); describeDb("publication receipt migration and lifecycle", () => { From d8241eccc04a28c22e474451f20ac7ea9bad4e62 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 03:31:32 +0000 Subject: [PATCH 29/34] Journal manual release deactivation --- scripts/deactivate-hosted-inference.ts | 27 +++++++++++++++----------- src/lib/release-job-rollout.ts | 6 ++++-- tests/private-worker-gates.test.ts | 6 +++++- tests/release-database-url.test.ts | 7 +++++++ 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/scripts/deactivate-hosted-inference.ts b/scripts/deactivate-hosted-inference.ts index f74e05eb..00ebadf8 100644 --- a/scripts/deactivate-hosted-inference.ts +++ b/scripts/deactivate-hosted-inference.ts @@ -1,8 +1,9 @@ import { closeDb, getPool } from "@/lib/db"; import { optionalEnv } from "@/lib/env"; import { - deactivateHostedInferenceRelease, - deactivatePublicationLifecycleRelease, + HOSTED_INFERENCE_FLEET_ACTIVE_CAPABILITY, + prepareManagedReleaseCapabilities, + PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY, } from "@/lib/release-job-rollout"; import { resolveDirectDatabaseUrl } from "./resolve-direct-database-url"; @@ -38,23 +39,27 @@ async function main(): Promise { console.log("managed release preparation skipped until the database schema exists"); return; } - const publicationLifecycle = schema.rows[0].publicationLifecycleReady - ? await deactivatePublicationLifecycleRelease(getPool()) - : { deactivated: false, parked: 0 }; + const preparation = await prepareManagedReleaseCapabilities( + getPool(), + releaseSha, + schema.rows[0].publicationLifecycleReady, + ); + const publicationLifecycleWasActive = preparation.capabilities.includes( + PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY, + ); const publicationLifecycleState = !schema.rows[0].publicationLifecycleReady ? "schema not installed" - : publicationLifecycle.deactivated + : publicationLifecycleWasActive ? "prior activation removed" : "already dark"; - const deactivated = await deactivateHostedInferenceRelease( - getPool(), - releaseSha, + const hostedInferenceWasActive = preparation.capabilities.includes( + HOSTED_INFERENCE_FLEET_ACTIVE_CAPABILITY, ); console.log( - `managed hosted inference prepared dark: ${deactivated ? "prior activation removed" : "already dark"}`, + `managed hosted inference prepared dark: ${hostedInferenceWasActive ? "prior activation removed" : "already dark"}; recovery generation=${preparation.generation}`, ); console.log( - `publication lifecycle prepared dark: ${publicationLifecycleState}; parked=${publicationLifecycle.parked}`, + `publication lifecycle prepared dark: ${publicationLifecycleState}`, ); } finally { await closeDb(); diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index ac13df5f..a3e45cdc 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -175,8 +175,10 @@ export async function deactivatePublicationLifecycleRelease( let releaseError: Error | undefined; let transactionOpen = false; try { - // Darken before draining so a legacy publisher that has not passed the - // capability check cannot begin while admitted operations finish. + // Managed release preparation records a durable recovery journal before + // this drain begins. Darken before draining so a legacy publisher that + // has not passed the capability check cannot begin while admitted + // operations finish; interruption remains fail-closed and recoverable. await client.query("BEGIN"); transactionOpen = true; const initial = await darkenPublicationLifecycle(client); diff --git a/tests/private-worker-gates.test.ts b/tests/private-worker-gates.test.ts index 06c36fd7..d03bd770 100644 --- a/tests/private-worker-gates.test.ts +++ b/tests/private-worker-gates.test.ts @@ -107,7 +107,11 @@ describe("private repository worker defense in depth", () => { expect(activation.indexOf("activatePublicationLifecycleRelease")).toBeLessThan( activation.indexOf("activateReleaseJobs"), ); - expect(deactivation).toContain("deactivatePublicationLifecycleRelease"); + expect(deactivation).toContain("prepareManagedReleaseCapabilities"); + expect(deactivation).not.toContain( + "deactivatePublicationLifecycleRelease", + ); + expect(deactivation).not.toContain("deactivateHostedInferenceRelease"); }); test("disabled hosted inference stops before reservation, config fetch, or CLI spawn", () => { diff --git a/tests/release-database-url.test.ts b/tests/release-database-url.test.ts index 94347e43..f198bf98 100644 --- a/tests/release-database-url.test.ts +++ b/tests/release-database-url.test.ts @@ -347,6 +347,13 @@ describe("release database connection", () => { ); expect(deactivationScript).toContain("resolveDirectDatabaseUrl"); expect(deactivationScript).toContain("publication_lifecycle_required_at"); + expect(deactivationScript).toContain("prepareManagedReleaseCapabilities"); + expect(deactivationScript).not.toContain( + "deactivateHostedInferenceRelease", + ); + expect(deactivationScript).not.toContain( + "deactivatePublicationLifecycleRelease", + ); expect(deactivationScript.indexOf("process.env.DATABASE_URL =")).toBeLessThan( deactivationScript.indexOf("getPool().query"), ); From ad682cac1bc29ce99f872341bbe195d55edac1ba Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 03:40:54 +0000 Subject: [PATCH 30/34] Recover orphaned release preparations --- .github/workflows/production-monitor.yml | 74 +++++++++++++-------- scripts/run-release-migrations.ts | 42 ++++++++++++ tests/publication-receipt-migration.test.ts | 8 +++ tests/release-database-url.test.ts | 15 ++++- 4 files changed, 108 insertions(+), 31 deletions(-) diff --git a/.github/workflows/production-monitor.yml b/.github/workflows/production-monitor.yml index 47e5e7b1..10d5b24d 100644 --- a/.github/workflows/production-monitor.yml +++ b/.github/workflows/production-monitor.yml @@ -24,8 +24,8 @@ permissions: concurrency: # Deployment-completion monitors keep independent workflow owners, while - # scheduled checks serialize together. The recovery job then joins the same - # bounded FIFO queue as deploys, so no pending recovery is replaced. + # scheduled checks serialize together. Every recovery attempt then joins the + # same bounded FIFO queue as deploys, so it observes an idle managed fleet. group: ${{ github.event_name == 'workflow_run' && format('production-monitor-deploy-{0}', github.event.workflow_run.id) || 'production-monitor' }} queue: max cancel-in-progress: false @@ -33,16 +33,17 @@ concurrency: jobs: release-recovery: name: Recover abandoned release preparation - if: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.conclusion != 'success' }} + if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion != 'success' }} concurrency: group: fly-deploy queue: max cancel-in-progress: false permissions: - actions: read contents: read runs-on: ubuntu-latest timeout-minutes: 6 + outputs: + recovered: ${{ steps.restore.outputs.recovered }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 @@ -50,7 +51,30 @@ jobs: bun-version: 1.3.14 - name: Install dependencies run: bun install --frozen-lockfile + - name: Find durable release preparation + id: preparation + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + run: | + set -euo pipefail + pending_releases="$(bun scripts/run-release-migrations.ts --pending-releases)" + if [[ -z "${pending_releases}" ]]; then + if bun scripts/run-release-migrations.ts --verify-clear >/dev/null 2>&1; then + echo "Release preparation is already active and clear." + echo "present=false" >> "${GITHUB_OUTPUT}" + exit 0 + fi + echo "Release capabilities are dark without a recovery journal." + exit 1 + fi + { + echo "present=true" + echo "targets<> "${GITHUB_OUTPUT}" - name: Install checksum-pinned flyctl + if: ${{ steps.preparation.outputs.present == 'true' }} env: FLYCTL_VERSION: 0.4.71 FLYCTL_LINUX_X86_64_SHA256: a782dceed173d215c000ab94e2b08623c22267edff6d90ebe3010b3f9b671dc2 @@ -69,29 +93,19 @@ jobs: install -m 0755 "${temporary_directory}/flyctl" "${RUNNER_TEMP}/flyctl" echo "${RUNNER_TEMP}" >> "${GITHUB_PATH}" - name: Restore only an unchanged prior fleet + id: restore + if: ${{ steps.preparation.outputs.present == 'true' }} env: FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} DATABASE_URL: ${{ secrets.DATABASE_URL }} - FAILED_DEPLOY_RUN_ID: ${{ github.event.workflow_run.id }} - GH_TOKEN: ${{ github.token }} - POSTIL_RELEASE_SHA: ${{ github.event.workflow_run.head_sha }} + EVENT_RELEASE_SHA: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || '' }} + PENDING_RELEASES: ${{ steps.preparation.outputs.targets }} run: | set -euo pipefail - latest_deploy_run_id=$(gh api \ - "repos/${GITHUB_REPOSITORY}/actions/workflows/deploy.yml/runs?per_page=1" \ - --jq '.workflow_runs[0].id') - if [[ "${latest_deploy_run_id}" != "${FAILED_DEPLOY_RUN_ID}" ]]; then - # queue:max preserves a separate recovery owner for every deploy - # completion. A superseded owner must not mutate the newer run's - # durable preparation or compensate against its fleet evidence. - echo "A newer deployment run owns release recovery." - exit 0 - fi - recovery_target_sha="${POSTIL_RELEASE_SHA}" - # Durable database state is authoritative. No journal plus both - # active fleet capabilities means no release preparation remains. - if bun scripts/run-release-migrations.ts --verify-clear >/dev/null 2>&1; then - echo "Release preparation is already active and clear." + mapfile -t recovery_targets <<<"${PENDING_RELEASES}" + recovery_target_sha="${recovery_targets[0]}" + if [[ -n "${EVENT_RELEASE_SHA}" && "${recovery_target_sha}" != "${EVENT_RELEASE_SHA}" ]]; then + echo "A newer release preparation owns recovery." exit 0 fi machines=$(flyctl machine list --app postil-web --json) @@ -121,9 +135,11 @@ jobs: exit 1 fi releases+=("${release}") - if [[ "${release}" == "${recovery_target_sha}" ]]; then - target_seen=1 - fi + for pending_release in "${recovery_targets[@]}"; do + if [[ "${release}" == "${pending_release}" ]]; then + target_seen=1 + fi + done done < <(jq -r '.[] | select( .config.metadata.fly_process_group == "web" or .config.metadata.fly_process_group == "worker" or @@ -135,11 +151,12 @@ jobs: exit 1 fi if [[ "${target_seen}" -ne 0 ]]; then - echo "The target release reached the managed fleet; capabilities remain dark." + echo "A pending release reached the managed fleet; capabilities remain dark." exit 1 fi POSTIL_RELEASE_SHA="${recovery_target_sha}" \ bun scripts/run-release-migrations.ts --compensate + echo "recovered=true" >> "${GITHUB_OUTPUT}" smoke: name: Smoke check production @@ -539,7 +556,7 @@ jobs: alert-key: ${{ (needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') && 'postil-release-recovery' || needs.smoke.result == 'failure' && 'postil-production-monitor' || 'postil-production-monitor-test' }} details: >- ${{ (needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') - && 'Failed deployment left release recovery unresolved. Run log:' + && 'Release preparation remains unresolved. Run log:' || needs.smoke.result == 'failure' && 'Production checks failed. Run log:' || 'Operator-requested test alert; production checks passed. Run log:' }} @@ -551,7 +568,8 @@ jobs: resolve-release-recovery: name: Resolve release recovery alert - if: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' }} + needs: release-recovery + if: ${{ always() && (needs.release-recovery.outputs.recovered == 'true' || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')) }} permissions: contents: read id-token: write diff --git a/scripts/run-release-migrations.ts b/scripts/run-release-migrations.ts index f05c11dd..cddae532 100644 --- a/scripts/run-release-migrations.ts +++ b/scripts/run-release-migrations.ts @@ -192,6 +192,43 @@ export async function releasePreparationCleared( } } +export async function pendingReleasePreparationTargets( + environment: Environment = process.env, +): Promise { + const databaseEnvironment = releaseMigrationEnvironment(environment); + const pool = new Pool({ connectionString: databaseEnvironment.DATABASE_URL }); + try { + const schema = await releaseSchemaState(pool); + if (!schema.hostedReady) return []; + const roots = await pool.query<{ name: string }>( + `SELECT name + FROM deployment_capabilities + WHERE name LIKE $1 + AND name LIKE '%:root' + ORDER BY activated_at DESC, name DESC`, + ["managed-release-preparation:%"], + ); + const targets: string[] = []; + const seen = new Set(); + for (const row of roots.rows) { + const match = row.name.match( + /^managed-release-preparation:([0-9a-f]{7,40}):[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}:root$/, + ); + if (!match) { + throw new Error("managed release preparation journal is malformed"); + } + const releaseSha = match[1]!; + if (!seen.has(releaseSha)) { + seen.add(releaseSha); + targets.push(releaseSha); + } + } + return targets; + } finally { + await pool.end(); + } +} + async function runReleaseDatabaseCommand( command: readonly string[], label: string, @@ -262,6 +299,11 @@ if (import.meta.main) { console.log("release preparation state is clear and active"); process.exit(0); } + if (process.argv[2] === "--pending-releases") { + const targets = await pendingReleasePreparationTargets(); + if (targets.length > 0) process.stdout.write(`${targets.join("\n")}\n`); + process.exit(0); + } const controller = new AbortController(); const interrupt = (signal: NodeJS.Signals) => { controller.abort(new Error(`release database preparation received ${signal}`)); diff --git a/tests/publication-receipt-migration.test.ts b/tests/publication-receipt-migration.test.ts index fc3b291c..e825f31d 100644 --- a/tests/publication-receipt-migration.test.ts +++ b/tests/publication-receipt-migration.test.ts @@ -35,6 +35,7 @@ import { } from "@/lib/release-job-rollout"; import { compensateReleasePreparation, + pendingReleasePreparationTargets, releasePreparationCleared, } from "../scripts/run-release-migrations"; @@ -1094,6 +1095,10 @@ describeDb("publication receipt migration and lifecycle", () => { ); await prepareManagedReleaseCapabilities(pool, releaseSha, true); + expect( + await pendingReleasePreparationTargets({ DATABASE_URL: TEST_URL! }), + ).toEqual([releaseSha]); + expect( await releasePreparationCleared({ DATABASE_URL: TEST_URL! }), ).toBe(false); @@ -1128,6 +1133,9 @@ describeDb("publication receipt migration and lifecycle", () => { expect( await releasePreparationCleared({ DATABASE_URL: TEST_URL! }), ).toBe(true); + expect( + await pendingReleasePreparationTargets({ DATABASE_URL: TEST_URL! }), + ).toEqual([]); }); test("same-release process compensation cannot overwrite a replacement generation", async () => { diff --git a/tests/release-database-url.test.ts b/tests/release-database-url.test.ts index f198bf98..417f141d 100644 --- a/tests/release-database-url.test.ts +++ b/tests/release-database-url.test.ts @@ -322,12 +322,21 @@ describe("release database connection", () => { queue: "max", "cancel-in-progress": false, }); - expect(productionMonitorWorkflow).toContain("latest_deploy_run_id"); expect(productionMonitorWorkflow).toContain( - "A newer deployment run owns release recovery.", + "bun scripts/run-release-migrations.ts --pending-releases", ); expect(productionMonitorWorkflow).toContain( - 'recovery_target_sha="${POSTIL_RELEASE_SHA}"', + "A newer release preparation owns recovery.", + ); + expect(productionMonitorWorkflow).not.toContain("latest_deploy_run_id"); + expect(productionMonitorWorkflow).toContain( + "github.event_name != 'workflow_run'", + ); + expect(productionMonitorWorkflow).toContain( + "needs.release-recovery.outputs.recovered == 'true'", + ); + expect(productionMonitorWorkflow).toContain( + 'recovery_target_sha="${recovery_targets[0]}"', ); expect(productionMonitorWorkflow).toContain( "needs: [smoke, release-recovery]", From dbef6803fc6fbf28b6be57f4579fe0e5f79e0b25 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 03:46:41 +0000 Subject: [PATCH 31/34] Clarify the startup probe oracle --- scripts/verify-server-observability-bundle.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/verify-server-observability-bundle.ts b/scripts/verify-server-observability-bundle.ts index f70b9995..4465dd25 100644 --- a/scripts/verify-server-observability-bundle.ts +++ b/scripts/verify-server-observability-bundle.ts @@ -55,6 +55,9 @@ const server = Bun.spawn( NEXT_TELEMETRY_DISABLED: "1", NODE_ENV: "production", PATH: process.env.PATH ?? "/usr/local/bin:/usr/bin:/bin", + // Do not pre-seed POSTIL_BOOT_PROBE_READY. The Node instrumentation + // hook sets it after startup registration, and the health header below + // proves that the built server actually ran that hook. POSTIL_BOOT_PROBE: bootProbe, POSTIL_PUBLIC_URL: "https://postil.invalid", POSTIL_SEALING_KEY: crypto.randomUUID().replaceAll("-", "").repeat(2), From 29669f113f3e2d6fa8adb5c0d828bedab59b50df Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 03:49:46 +0000 Subject: [PATCH 32/34] Document release recovery invariants --- .github/workflows/production-monitor.yml | 2 ++ src/lib/release-job-rollout.ts | 12 +++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/production-monitor.yml b/.github/workflows/production-monitor.yml index 10d5b24d..0e789a82 100644 --- a/.github/workflows/production-monitor.yml +++ b/.github/workflows/production-monitor.yml @@ -127,6 +127,8 @@ jobs: releases=() target_seen=0 while IFS= read -r id; do + # The flyctl execution timeout bounds every individual machine + # probe; workflow timeout remains the independent outer bound. release=$(flyctl machine exec "${id}" \ "bun -e 'process.stdout.write(process.env.POSTIL_RELEASE_SHA ?? \"\")'" \ --app postil-web --timeout 15 2>/dev/null) diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index a3e45cdc..27b01161 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -176,9 +176,11 @@ export async function deactivatePublicationLifecycleRelease( let transactionOpen = false; try { // Managed release preparation records a durable recovery journal before - // this drain begins. Darken before draining so a legacy publisher that - // has not passed the capability check cannot begin while admitted - // operations finish; interruption remains fail-closed and recoverable. + // this drain begins. The first commit removes the active capability, so + // the database trigger parks every new gate job before the exclusive lock + // is requested. Admitted legacy operations can then drain without a gap + // that lets another active publisher enter; interruption remains + // fail-closed and recoverable. await client.query("BEGIN"); transactionOpen = true; const initial = await darkenPublicationLifecycle(client); @@ -224,6 +226,8 @@ export async function deactivatePublicationLifecycleRelease( ); } } + // A successful rollback leaves the client reusable. releaseError is set + // only when BEGIN or rollback leaves the backend state uncertain. throw primaryError; } finally { client.release(releaseError); @@ -411,6 +415,8 @@ export async function activatePublicationLifecycleRelease( "publication lifecycle activation and rollback failed", ); } + // A successful rollback leaves the client reusable. releaseError is set + // only when BEGIN or rollback leaves the backend state uncertain. throw primaryError; } finally { client.release(releaseError); From 2a4f7fdc9ca24faf7275391717f94b9731b3768b Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 03:53:41 +0000 Subject: [PATCH 33/34] Fence release recovery ownership --- .github/workflows/production-monitor.yml | 25 +++++--- scripts/deactivate-hosted-inference.ts | 72 +----------------------- tests/private-worker-gates.test.ts | 3 +- tests/release-database-url.test.ts | 15 ++--- 4 files changed, 30 insertions(+), 85 deletions(-) diff --git a/.github/workflows/production-monitor.yml b/.github/workflows/production-monitor.yml index 0e789a82..460af0ed 100644 --- a/.github/workflows/production-monitor.yml +++ b/.github/workflows/production-monitor.yml @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 6 outputs: - recovered: ${{ steps.restore.outputs.recovered }} + clear: ${{ steps.verified.outputs.clear }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 @@ -106,14 +106,15 @@ jobs: recovery_target_sha="${recovery_targets[0]}" if [[ -n "${EVENT_RELEASE_SHA}" && "${recovery_target_sha}" != "${EVENT_RELEASE_SHA}" ]]; then echo "A newer release preparation owns recovery." + echo "superseded=true" >> "${GITHUB_OUTPUT}" exit 0 fi machines=$(flyctl machine list --app postil-web --json) - managed_count=$(jq -r '[.[] | select( - .config.metadata.fly_process_group == "web" or - .config.metadata.fly_process_group == "worker" or - .config.metadata.fly_process_group == "monitor" - )] | length' <<<"${machines}") + if ! fleet_summary=$(jq -ce -f scripts/verify-managed-fleet.jq <<<"${machines}"); then + echo "Managed fleet topology is invalid; release capabilities remain dark." + exit 1 + fi + managed_count=$(jq -r '.managed_count' <<<"${fleet_summary}") started_count=$(jq -r '[.[] | select( .state == "started" and (.config.metadata.fly_process_group == "web" or @@ -158,7 +159,15 @@ jobs: fi POSTIL_RELEASE_SHA="${recovery_target_sha}" \ bun scripts/run-release-migrations.ts --compensate - echo "recovered=true" >> "${GITHUB_OUTPUT}" + - name: Verify release recovery is clear + id: verified + if: ${{ always() && !cancelled() && steps.preparation.outcome == 'success' && steps.restore.outcome != 'failure' && steps.restore.outputs.superseded != 'true' }} + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + run: | + set -euo pipefail + bun scripts/run-release-migrations.ts --verify-clear + echo "clear=true" >> "${GITHUB_OUTPUT}" smoke: name: Smoke check production @@ -571,7 +580,7 @@ jobs: resolve-release-recovery: name: Resolve release recovery alert needs: release-recovery - if: ${{ always() && (needs.release-recovery.outputs.recovered == 'true' || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')) }} + if: ${{ always() && (needs.release-recovery.outputs.clear == 'true' || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')) }} permissions: contents: read id-token: write diff --git a/scripts/deactivate-hosted-inference.ts b/scripts/deactivate-hosted-inference.ts index 00ebadf8..4da47a62 100644 --- a/scripts/deactivate-hosted-inference.ts +++ b/scripts/deactivate-hosted-inference.ts @@ -1,69 +1,3 @@ -import { closeDb, getPool } from "@/lib/db"; -import { optionalEnv } from "@/lib/env"; -import { - HOSTED_INFERENCE_FLEET_ACTIVE_CAPABILITY, - prepareManagedReleaseCapabilities, - PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY, -} from "@/lib/release-job-rollout"; -import { resolveDirectDatabaseUrl } from "./resolve-direct-database-url"; - -async function main(): Promise { - try { - const releaseSha = optionalEnv("POSTIL_RELEASE_SHA"); - if (!releaseSha) { - console.log("managed hosted inference preparation skipped outside a release image"); - return; - } - process.env.DATABASE_URL = resolveDirectDatabaseUrl({ - databaseUrl: process.env.DATABASE_URL ?? "", - directDatabaseUrl: process.env.POSTIL_DIRECT_DATABASE_URL, - }); - delete process.env.POSTIL_DIRECT_DATABASE_URL; - const schema = await getPool().query<{ - hostedReady: boolean; - publicationLifecycleReady: boolean; - }>( - `SELECT - to_regclass('public.deployment_capabilities') IS NOT NULL AS "hostedReady", - to_regclass('public.deployment_capabilities') IS NOT NULL - AND to_regclass('public.jobs') IS NOT NULL - AND EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_schema = 'public' - AND table_name = 'reviews' - AND column_name = 'publication_lifecycle_required_at' - ) AS "publicationLifecycleReady"`, - ); - if (schema.rows[0]?.hostedReady !== true) { - console.log("managed release preparation skipped until the database schema exists"); - return; - } - const preparation = await prepareManagedReleaseCapabilities( - getPool(), - releaseSha, - schema.rows[0].publicationLifecycleReady, - ); - const publicationLifecycleWasActive = preparation.capabilities.includes( - PUBLICATION_LIFECYCLE_FLEET_ACTIVE_CAPABILITY, - ); - const publicationLifecycleState = !schema.rows[0].publicationLifecycleReady - ? "schema not installed" - : publicationLifecycleWasActive - ? "prior activation removed" - : "already dark"; - const hostedInferenceWasActive = preparation.capabilities.includes( - HOSTED_INFERENCE_FLEET_ACTIVE_CAPABILITY, - ); - console.log( - `managed hosted inference prepared dark: ${hostedInferenceWasActive ? "prior activation removed" : "already dark"}; recovery generation=${preparation.generation}`, - ); - console.log( - `publication lifecycle prepared dark: ${publicationLifecycleState}`, - ); - } finally { - await closeDb(); - } -} - -if (import.meta.main) await main(); +throw new Error( + "standalone release deactivation is unsupported; use the managed deployment workflow so preparation and recovery share one owner queue", +); diff --git a/tests/private-worker-gates.test.ts b/tests/private-worker-gates.test.ts index d03bd770..621f4634 100644 --- a/tests/private-worker-gates.test.ts +++ b/tests/private-worker-gates.test.ts @@ -107,7 +107,8 @@ describe("private repository worker defense in depth", () => { expect(activation.indexOf("activatePublicationLifecycleRelease")).toBeLessThan( activation.indexOf("activateReleaseJobs"), ); - expect(deactivation).toContain("prepareManagedReleaseCapabilities"); + expect(deactivation).toContain("standalone release deactivation is unsupported"); + expect(deactivation).not.toContain("prepareManagedReleaseCapabilities"); expect(deactivation).not.toContain( "deactivatePublicationLifecycleRelease", ); diff --git a/tests/release-database-url.test.ts b/tests/release-database-url.test.ts index 417f141d..450b1a39 100644 --- a/tests/release-database-url.test.ts +++ b/tests/release-database-url.test.ts @@ -333,7 +333,10 @@ describe("release database connection", () => { "github.event_name != 'workflow_run'", ); expect(productionMonitorWorkflow).toContain( - "needs.release-recovery.outputs.recovered == 'true'", + "needs.release-recovery.outputs.clear == 'true'", + ); + expect(productionMonitorWorkflow).toContain( + "jq -ce -f scripts/verify-managed-fleet.jq", ); expect(productionMonitorWorkflow).toContain( 'recovery_target_sha="${recovery_targets[0]}"', @@ -354,17 +357,15 @@ describe("release database connection", () => { expect(productionMonitorWorkflow).toContain( "bun scripts/run-release-migrations.ts --verify-clear", ); - expect(deactivationScript).toContain("resolveDirectDatabaseUrl"); - expect(deactivationScript).toContain("publication_lifecycle_required_at"); - expect(deactivationScript).toContain("prepareManagedReleaseCapabilities"); + expect(deactivationScript).toContain( + "standalone release deactivation is unsupported", + ); + expect(deactivationScript).not.toContain("prepareManagedReleaseCapabilities"); expect(deactivationScript).not.toContain( "deactivateHostedInferenceRelease", ); expect(deactivationScript).not.toContain( "deactivatePublicationLifecycleRelease", ); - expect(deactivationScript.indexOf("process.env.DATABASE_URL =")).toBeLessThan( - deactivationScript.indexOf("getPool().query"), - ); }); }); From 91461ff1b71b6cb72faf0663b4c97e2102b72b25 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 03:58:04 +0000 Subject: [PATCH 34/34] Verify topology before deploy recovery --- .github/workflows/deploy.yml | 10 +++++----- tests/release-database-url.test.ts | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8f3a38fe..18e90465 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -333,11 +333,11 @@ jobs: run: | set -euo pipefail machines=$(flyctl machine list --app postil-web --json) - managed_count=$(jq -r '[.[] | select( - .config.metadata.fly_process_group == "web" or - .config.metadata.fly_process_group == "worker" or - .config.metadata.fly_process_group == "monitor" - )] | length' <<<"${machines}") + if ! fleet_summary=$(jq -ce -f scripts/verify-managed-fleet.jq <<<"${machines}"); then + echo "Managed fleet topology is invalid; release capabilities remain dark." + exit 1 + fi + managed_count=$(jq -r '.managed_count' <<<"${fleet_summary}") started_count=$(jq -r '[.[] | select( .state == "started" and (.config.metadata.fly_process_group == "web" or diff --git a/tests/release-database-url.test.ts b/tests/release-database-url.test.ts index 450b1a39..412a04da 100644 --- a/tests/release-database-url.test.ts +++ b/tests/release-database-url.test.ts @@ -300,6 +300,9 @@ describe("release database connection", () => { expect(deployWorkflow).toContain( "Restore capabilities when release preparation failed before replacement", ); + expect( + deployWorkflow.split("jq -ce -f scripts/verify-managed-fleet.jq").length - 1, + ).toBeGreaterThanOrEqual(2); expect(deployWorkflow).toContain("bun scripts/run-release-migrations.ts --compensate"); expect(productionMonitorWorkflow).toContain('workflows: ["deploy"]'); expect(deployWorkflowConfig.concurrency).toEqual({