diff --git a/.env.example b/.env.example index e50751c1..fe0d3fbc 100644 --- a/.env.example +++ b/.env.example @@ -27,11 +27,11 @@ POSTIL_SESSION_SECRET= GITHUB_WEBHOOK_SECRET= # Optional iLert outbound alert receiver. Generate a random password with -# `openssl rand -hex 32`, set it here, and configure the iLert webhook URL as -# https://postil-ilert:@/api/webhooks/ilert. -# Select alert-created, alert-assigned, alert-auto-escalated, -# alert-auto-resolved, alert-acknowledged, alert-rejected, alert-raised, and -# alert-resolved as trigger events. Leave unset to make the endpoint return 404. +# `openssl rand -hex 32`, set it here, and configure a Webhook alert action for +# `https:///api/webhooks/ilert`. Send an `Authorization` header with +# HTTP Basic credentials using `postil-ilert` as the username and this value as +# the password. Select alert-created, alert-acknowledged, alert-comment-added, +# and alert-resolved as trigger events. Leave unset to return 404. POSTIL_ILERT_WEBHOOK_SECRET= # Numeric GitHub ids allowed to use operator-only surfaces. The alert stream diff --git a/.github/workflows/production-monitor.yml b/.github/workflows/production-monitor.yml index 460af0ed..b35a2059 100644 --- a/.github/workflows/production-monitor.yml +++ b/.github/workflows/production-monitor.yml @@ -535,6 +535,59 @@ jobs: fi fi + alert-stream: + name: Verify operator alert stream + needs: smoke + if: ${{ inputs.test_alert == true }} + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.3.14 + - name: Load iLert management secret from Infisical + uses: Infisical/secrets-action@77ab1f4ccd183a543cb5b42435fbd181189f4995 # v1.0.16 + with: + method: oidc + identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} + project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }} + env-slug: prod + domain: https://eu.infisical.com + secret-path: / + secret-name: ILERT_API_KEY + - name: Load iLert integration secret from Infisical + uses: Infisical/secrets-action@77ab1f4ccd183a543cb5b42435fbd181189f4995 # v1.0.16 + with: + method: oidc + identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} + project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }} + env-slug: prod + domain: https://eu.infisical.com + secret-path: /postil + secret-name: ILERT_INTEGRATION_KEY + - name: Load Postil webhook secret from Infisical + uses: Infisical/secrets-action@77ab1f4ccd183a543cb5b42435fbd181189f4995 # v1.0.16 + with: + method: oidc + identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} + project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }} + env-slug: prod + domain: https://eu.infisical.com + secret-path: /postil + secret-name: POSTIL_ILERT_WEBHOOK_SECRET + - name: Preview iLert alert-stream reconciliation + env: + POSTIL_ILERT_ALERT_SOURCE_ID: ${{ vars.POSTIL_ILERT_ALERT_SOURCE_ID }} + run: bun run scripts/reconcile-ilert-alert-stream.ts --dry-run + - name: Reconcile, deliver, and resolve the unique iLert canary + env: + POSTIL_ILERT_ALERT_SOURCE_ID: ${{ vars.POSTIL_ILERT_ALERT_SOURCE_ID }} + run: bun run scripts/reconcile-ilert-alert-stream.ts --canary + # Alert delivery is owned by the external alerting service (ilert). The # platform and this workflow only report events; paging, escalation, and # deduplication happen there. A stable alertKey groups consecutive failing @@ -542,7 +595,7 @@ jobs: notify: name: Raise external alert needs: [smoke, release-recovery] - if: ${{ always() && (needs.smoke.result == 'failure' || needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled' || inputs.test_alert == true) }} + if: ${{ always() && (needs.smoke.result == 'failure' || needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') }} permissions: contents: read id-token: write @@ -563,19 +616,17 @@ jobs: uses: ./.github/actions/ilert-event with: event-type: ALERT - summary: ${{ (needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') && 'Postil release recovery failed' || needs.smoke.result == 'failure' && 'Postil production monitor failed' || 'Postil production monitor test alert' }} - alert-key: ${{ (needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') && 'postil-release-recovery' || needs.smoke.result == 'failure' && 'postil-production-monitor' || 'postil-production-monitor-test' }} + summary: ${{ (needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') && 'Postil release recovery failed' || 'Postil production monitor failed' }} + alert-key: ${{ (needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') && 'postil-release-recovery' || 'postil-production-monitor' }} details: >- ${{ (needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') && 'Release preparation remains unresolved. Run log:' - || needs.smoke.result == 'failure' - && 'Production checks failed. Run log:' - || 'Operator-requested test alert; production checks passed. Run log:' }} + || 'Production checks failed. Run log:' }} ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} # A routine monitor failure records an alerting gap without masking - # the original signal. Recovery failure and test events require - # delivery because they validate the fail-safe notification path. - require-delivery: ${{ inputs.test_alert == true || needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled' }} + # the original signal. Recovery failure requires delivery because it + # validates the fail-safe notification path. + require-delivery: ${{ needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled' }} resolve-release-recovery: name: Resolve release recovery alert @@ -613,6 +664,9 @@ jobs: summary: Postil release recovery cleared alert-key: postil-release-recovery + # The alert-stream job resolves its unique canary before it succeeds. This + # resolver owns only the stable production-monitor alert key, so a manual + # canary must not close a production incident. resolve: name: Resolve external alert needs: smoke diff --git a/scripts/reconcile-ilert-alert-stream.ts b/scripts/reconcile-ilert-alert-stream.ts new file mode 100644 index 00000000..252a2b07 --- /dev/null +++ b/scripts/reconcile-ilert-alert-stream.ts @@ -0,0 +1,502 @@ +#!/usr/bin/env bun + +import { randomUUID } from "node:crypto"; + +const API_BASE = "https://api.ilert.com/api"; +const WEBHOOK_URL = "https://postil.dev/api/webhooks/ilert"; +const ACTION_NAME = "Postil operator alert stream"; +const REQUEST_TIMEOUT_MS = 20_000; +const CANARY_ATTEMPTS = 12; +const CANARY_RETRY_MS = 5_000; + +export const ALERT_TRIGGER_TYPES = [ + "alert-created", + "alert-acknowledged", + "alert-comment-added", + "alert-resolved", +] as const; + +type Json = Record; +export type Fetch = ( + input: string | URL | Request, + init?: RequestInit, +) => Promise; +type Sleep = (milliseconds: number) => Promise; +type Operation = "create" | "update" | "unchanged"; + +interface ReconcileOptions { + apiKey: string; + sourceId: number; + webhookSecret: string; + dryRun?: boolean; + fetchFn?: Fetch; +} + +interface CanaryOptions { + actionId: string; + apiKey: string; + integrationKey: string; + fetchFn?: Fetch; + sleep?: Sleep; + runId?: string; + runAttempt?: string; +} + +export interface ReconcileResult { + actionId: string | null; + operation: Operation; +} + +export function desiredAlertAction(source: Json, secret: string): Json { + const authorization = Buffer.from(`postil-ilert:${secret}`, "utf8").toString( + "base64", + ); + return { + alertSources: [source], + connectorType: "webhook", + name: ACTION_NAME, + triggerMode: "AUTOMATIC", + triggerTypes: [...ALERT_TRIGGER_TYPES], + params: { + webhookUrl: WEBHOOK_URL, + headers: [{ key: "Authorization", value: `Basic ${authorization}` }], + }, + }; +} + +export function equivalentAlertAction(actual: Json, desired: Json): boolean { + const left = object(actual.params); + const right = object(desired.params); + return ( + actual.name === desired.name && + actual.connectorType === desired.connectorType && + actual.triggerMode === desired.triggerMode && + sameSet(strings(actual.triggerTypes), strings(desired.triggerTypes)) && + sameSet(relationIds(actual.alertSources), relationIds(desired.alertSources)) && + left?.webhookUrl === right?.webhookUrl && + sameSet(headers(left?.headers), headers(right?.headers)) + ); +} + +export async function reconcileIlertAlertAction( + options: ReconcileOptions, +): Promise { + const fetchFn = options.fetchFn ?? fetch; + const source = alertSource( + await management(fetchFn, options.apiKey, `/alert-sources/${options.sourceId}`), + options.sourceId, + ); + const desired = desiredAlertAction(source, options.webhookSecret); + const listed = await listActions(fetchFn, options.apiKey); + const actions: Json[] = []; + for (let index = 0; index < listed.length; index += 8) { + actions.push( + ...(await Promise.all( + listed.slice(index, index + 8).map(async (item) => + requireObject( + await management( + fetchFn, + options.apiKey, + `/alert-actions/${encodeURIComponent(actionId(item))}`, + ), + "iLert returned an invalid alert action", + ), + ), + )), + ); + } + const candidates = actions.filter((action) => { + const params = object(action.params); + return action.name === ACTION_NAME || params?.webhookUrl === WEBHOOK_URL; + }); + if ( + candidates.some( + (action) => + action.connectorType !== "webhook" || + !sameSet(relationIds(action.alertSources), [options.sourceId]), + ) + ) { + throw new Error( + "A conflicting Postil alert action exists; refusing to change its type or source scope", + ); + } + if (candidates.length > 1) { + throw new Error( + "Multiple Postil webhook alert actions exist; refusing to choose or delete one", + ); + } + + const existing = candidates[0]; + const operation: Operation = !existing + ? "create" + : equivalentAlertAction(existing, desired) + ? "unchanged" + : "update"; + if (operation === "unchanged") { + return { actionId: actionId(existing!), operation }; + } + if (options.dryRun) { + return { actionId: existing ? actionId(existing) : null, operation }; + } + + const id = existing ? actionId(existing) : null; + const result = requireObject( + await management( + fetchFn, + options.apiKey, + id ? `/alert-actions/${encodeURIComponent(id)}` : "/alert-actions", + { + method: id ? "PUT" : "POST", + body: JSON.stringify(id ? { ...desired, id } : desired), + }, + ), + "iLert returned an invalid alert action", + ); + const resultId = actionId(result); + const confirmed = requireObject( + await management( + fetchFn, + options.apiKey, + `/alert-actions/${encodeURIComponent(resultId)}`, + ), + "iLert returned an invalid alert action", + ); + if (!equivalentAlertAction(confirmed, desired)) { + throw new Error("iLert did not retain the reconciled alert action"); + } + return { actionId: resultId, operation }; +} + +export async function verifyIlertAlertStreamCanary( + options: CanaryOptions, +): Promise { + const fetchFn = options.fetchFn ?? fetch; + const sleep = options.sleep ?? Bun.sleep; + const startedAt = new Date(Date.now() - 5_000).toISOString(); + const key = [ + "postil-operator-alert-stream-canary", + options.runId ?? "local", + options.runAttempt ?? "1", + randomUUID(), + ].join("-"); + let accepted = false; + let resolveSent = false; + let failure: unknown; + try { + await event(fetchFn, options.integrationKey, "ALERT", key); + accepted = true; + const created = await waitForDelivery({ + ...options, + fetchFn, + key, + sleep, + startedAt, + }); + await event(fetchFn, options.integrationKey, "RESOLVE", key); + resolveSent = true; + await waitForDelivery({ + ...options, + alertId: created.alertId, + fetchFn, + key, + minimumDeliveries: created.deliveries + 1, + requiredStatus: "RESOLVED", + sleep, + startedAt, + }); + } catch (error) { + failure = error; + throw error; + } finally { + if (accepted && !resolveSent) { + try { + await event(fetchFn, options.integrationKey, "RESOLVE", key); + } catch (error) { + if (!failure) throw error; + } + } + } +} + +interface WaitOptions extends CanaryOptions { + alertId?: string; + fetchFn: Fetch; + key: string; + minimumDeliveries?: number; + requiredStatus?: string; + sleep: Sleep; + startedAt: string; +} + +async function waitForDelivery( + options: WaitOptions, +): Promise<{ alertId: string; deliveries: number }> { + for (let attempt = 0; attempt < CANARY_ATTEMPTS; attempt += 1) { + const alerts = await management( + options.fetchFn, + options.apiKey, + `/alerts?from=${encodeURIComponent(options.startedAt)}&max-results=100`, + ); + if (!Array.isArray(alerts)) { + throw new Error("iLert returned an invalid alert list during the canary"); + } + const alert = object( + alerts.find((item) => object(item)?.alertKey === options.key), + ); + const id = positiveId(alert?.id); + if (id && (!options.alertId || options.alertId === id)) { + const value = await management( + options.fetchFn, + options.apiKey, + `/alerts/${encodeURIComponent(id)}/actions`, + ); + const actions = Array.isArray(value) ? value : [value]; + if (actions.some((item) => !object(item))) { + throw new Error("iLert returned invalid action history during the canary"); + } + const deliveries = actions + .filter( + (item) => opaqueId(object(item)?.alertActionId) === options.actionId, + ) + .flatMap((item) => { + const history = object(item)?.history; + return Array.isArray(history) ? history : []; + }) + .filter((item) => object(item)?.success === true).length; + if ( + deliveries >= (options.minimumDeliveries ?? 1) && + (!options.requiredStatus || alert?.status === options.requiredStatus) + ) { + return { alertId: id, deliveries }; + } + } + if (attempt + 1 < CANARY_ATTEMPTS) await options.sleep(CANARY_RETRY_MS); + } + throw new Error("iLert did not confirm successful Postil webhook delivery"); +} + +async function event( + fetchFn: Fetch, + integrationKey: string, + eventType: "ALERT" | "RESOLVE", + alertKey: string, +): Promise { + const response = await request(fetchFn, `${API_BASE}/events`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + integrationKey, + eventType, + summary: + eventType === "ALERT" + ? "Postil operator alert stream canary" + : "Postil operator alert stream canary resolved", + ...(eventType === "ALERT" + ? { + details: + "GitHub Actions is verifying the Postil operator notification path.", + priority: "HIGH", + } + : {}), + alertKey, + }), + }); + if (!response.ok) { + throw new Error(`iLert event request failed with HTTP ${response.status}`); + } +} + +async function management( + fetchFn: Fetch, + apiKey: string, + path: string, + init: RequestInit = {}, +): Promise { + const response = await request(fetchFn, `${API_BASE}${path}`, { + ...init, + headers: { + accept: "application/json", + authorization: apiKey, + ...(init.body ? { "content-type": "application/json" } : {}), + }, + }); + if (!response.ok) { + throw new Error(`iLert management request failed with HTTP ${response.status}`); + } + try { + return await response.json(); + } catch { + throw new Error("iLert returned invalid JSON"); + } +} + +function request(fetchFn: Fetch, url: string, init: RequestInit): Promise { + return fetchFn(url, { ...init, signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) }); +} + +async function listActions(fetchFn: Fetch, apiKey: string): Promise { + const actions: Json[] = []; + for (let start = 0; start <= 1_000; start += 100) { + const page = await management( + fetchFn, + apiKey, + `/alert-actions?start-index=${start}&max-results=100`, + ); + if (!Array.isArray(page) || page.some((item) => !object(item))) { + throw new Error("iLert returned an invalid alert-action list"); + } + actions.push(...(page as Json[])); + if (page.length < 100) return actions; + } + throw new Error("iLert alert-action pagination exceeded the safety bound"); +} + +function alertSource(value: unknown, expectedId: number): Json { + const source = requireObject(value, "iLert returned an invalid alert source"); + const policy = object(source.escalationPolicy); + const id = positiveNumber(source.id); + if ( + id !== expectedId || + !nonempty(source.name) || + !nonempty(source.integrationType) || + !nonempty(policy?.name) || + !Array.isArray(policy?.escalationRules) + ) { + throw new Error("iLert returned an invalid alert source"); + } + const policyId = positiveNumber(policy.id); + return { + id, + name: source.name, + integrationType: source.integrationType, + escalationPolicy: { + ...(policyId ? { id: policyId } : {}), + name: policy.name, + escalationRules: policy.escalationRules, + }, + }; +} + +function actionId(value: unknown): string { + const id = opaqueId(object(value)?.id); + if (id) return id; + throw new Error("iLert returned an alert action without an identity"); +} + +function opaqueId(value: unknown): string | null { + if (typeof value === "number" && Number.isSafeInteger(value) && value > 0) { + return String(value); + } + if ( + typeof value === "string" && + value.length <= 128 && + /^[A-Za-z0-9._:-]+$/u.test(value) + ) { + return value; + } + return null; +} + +function positiveId(value: unknown): string | null { + const id = positiveNumber(value); + return id ? String(id) : null; +} + +function positiveNumber(value: unknown): number | null { + const parsed = + typeof value === "string" && /^[1-9][0-9]*$/u.test(value) + ? Number(value) + : value; + return typeof parsed === "number" && Number.isSafeInteger(parsed) && parsed > 0 + ? parsed + : null; +} + +function relationIds(value: unknown): number[] { + return Array.isArray(value) + ? value.flatMap((item) => { + const id = positiveNumber(object(item)?.id); + return id ? [id] : []; + }) + : []; +} + +function headers(value: unknown): string[] { + return Array.isArray(value) + ? value.flatMap((item) => { + const header = object(item); + return typeof header?.key === "string" && typeof header.value === "string" + ? [`${header.key.toLowerCase()}:${header.value}`] + : []; + }) + : []; +} + +function strings(value: unknown): string[] { + return Array.isArray(value) && value.every((item) => typeof item === "string") + ? value + : []; +} + +function sameSet(left: T[], right: T[]): boolean { + return [...left].sort().join("\0") === [...right].sort().join("\0"); +} + +function object(value: unknown): Json | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Json) + : null; +} + +function requireObject(value: unknown, message: string): Json { + const result = object(value); + if (!result) throw new Error(message); + return result; +} + +function nonempty(value: unknown): value is string { + return typeof value === "string" && value.length > 0; +} + +function environment(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} + +async function main(): Promise { + const args = process.argv.slice(2); + if (args.some((arg) => arg !== "--dry-run" && arg !== "--canary")) { + throw new Error("usage: reconcile-ilert-alert-stream.ts [--dry-run] [--canary]"); + } + const dryRun = args.includes("--dry-run"); + const canary = args.includes("--canary"); + if (dryRun && canary) throw new Error("--canary cannot be combined with --dry-run"); + const apiKey = environment("ILERT_API_KEY"); + const sourceId = Number(environment("POSTIL_ILERT_ALERT_SOURCE_ID")); + if (!Number.isSafeInteger(sourceId) || sourceId <= 0) { + throw new Error("POSTIL_ILERT_ALERT_SOURCE_ID must be a positive integer"); + } + const result = await reconcileIlertAlertAction({ + apiKey, + sourceId, + webhookSecret: environment("POSTIL_ILERT_WEBHOOK_SECRET"), + dryRun, + }); + console.log( + `iLert alert-stream reconciliation${dryRun ? " plan" : ""}: ${result.operation}`, + ); + if (canary) { + if (!result.actionId) throw new Error("the reconciled alert action has no identity"); + await verifyIlertAlertStreamCanary({ + actionId: result.actionId, + apiKey, + integrationKey: environment("ILERT_INTEGRATION_KEY"), + runId: process.env.GITHUB_RUN_ID, + runAttempt: process.env.GITHUB_RUN_ATTEMPT, + }); + console.log("iLert confirmed successful Postil webhook delivery"); + } +} + +if (import.meta.main) await main(); diff --git a/tests/health.test.ts b/tests/health.test.ts index 347f5178..daa79c5b 100644 --- a/tests/health.test.ts +++ b/tests/health.test.ts @@ -192,6 +192,37 @@ describe("/api/health/monitor", () => { }); describe("production monitor workflow", () => { + test("owns a bounded manual alert-stream canary and resolves test alerts", async () => { + const source = await readFile( + new URL("../.github/workflows/production-monitor.yml", import.meta.url), + "utf8", + ); + const workflow = parse(source) as { + jobs: Record< + string, + { + if?: string; + needs?: string; + steps?: Array<{ name?: string; run?: string }>; + } + >; + }; + const alertStream = workflow.jobs["alert-stream"]; + expect(alertStream?.needs).toBe("smoke"); + expect(alertStream?.if).toContain("inputs.test_alert == true"); + expect(alertStream?.steps?.map((step) => step.name)).toContain( + "Preview iLert alert-stream reconciliation", + ); + expect(alertStream?.steps?.map((step) => step.name)).toContain( + "Reconcile, deliver, and resolve the unique iLert canary", + ); + expect(workflow.jobs.notify?.if).not.toContain("inputs.test_alert"); + expect(workflow.jobs.resolve?.if).toContain( + "needs.smoke.result == 'success'", + ); + expect(workflow.jobs.resolve?.if).toContain("inputs.test_alert != true"); + }); + test("enforces monitor health, collection, delivery, failure, and stuck-pass signals", async () => { const source = await readFile( new URL("../.github/workflows/production-monitor.yml", import.meta.url), diff --git a/tests/reconcile-ilert-alert-stream.test.ts b/tests/reconcile-ilert-alert-stream.test.ts new file mode 100644 index 00000000..25a4dfdf --- /dev/null +++ b/tests/reconcile-ilert-alert-stream.test.ts @@ -0,0 +1,281 @@ +import { describe, expect, test } from "bun:test"; + +import { + ALERT_TRIGGER_TYPES, + desiredAlertAction, + equivalentAlertAction, + type Fetch, + reconcileIlertAlertAction, + verifyIlertAlertStreamCanary, +} from "../scripts/reconcile-ilert-alert-stream"; + +const API_KEY = "test-api-key"; +const SOURCE_ID = 42; +const WEBHOOK_SECRET = "test-webhook-secret-with-at-least-32-bytes"; +const SOURCE = { + id: SOURCE_ID, + name: "Postil test source", + integrationType: "API", + escalationPolicy: { + id: 7, + name: "Test escalation", + escalationRules: [], + }, +}; + +describe("iLert alert-stream reconciliation", () => { + test("builds the bounded automatic webhook action", () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET); + expect(desired).toMatchObject({ + alertSources: [SOURCE], + connectorType: "webhook", + name: "Postil operator alert stream", + triggerMode: "AUTOMATIC", + triggerTypes: [...ALERT_TRIGGER_TYPES], + params: { + webhookUrl: "https://postil.dev/api/webhooks/ilert", + headers: [{ key: "Authorization" }], + }, + }); + expect(JSON.stringify(desired)).not.toContain(WEBHOOK_SECRET); + }); + + test("compares trigger and header sets without depending on order or case", () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET); + const actual = structuredClone(desired) as Record; + actual.id = "42"; + actual.triggerTypes = [...ALERT_TRIGGER_TYPES].reverse(); + const params = actual.params as { headers: Array<{ key: string; value: string }> }; + params.headers[0]!.key = "authorization"; + expect(equivalentAlertAction(actual, desired)).toBe(true); + }); + + test("dry-run reports create without mutating", async () => { + const requests: Request[] = []; + const result = await reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + dryRun: true, + fetchFn: fakeFetch(requests, [Response.json(SOURCE), Response.json([])]), + }); + expect(result).toEqual({ actionId: null, operation: "create" }); + expect(requests).toHaveLength(2); + expect(requests[0]!.method).toBe("GET"); + }); + + test("rejects an incomplete source instead of sending a lossy relation", async () => { + await expect( + reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + dryRun: true, + fetchFn: fakeFetch([], [ + Response.json({ id: SOURCE_ID, name: SOURCE.name }), + ]), + }), + ).rejects.toThrow("invalid alert source"); + }); + + test("checks every action page before planning a create", async () => { + const requests: Request[] = []; + const firstPage = Array.from({ length: 100 }, (_, index) => ({ + id: String(index + 1), + })); + const responses = [ + Response.json(SOURCE), + Response.json(firstPage), + Response.json([]), + ...firstPage.map(({ id }) => + Response.json({ + id, + name: `Unrelated action ${id}`, + connectorType: "webhook", + alertSources: [{ id: SOURCE_ID + 1 }], + params: { webhookUrl: `https://example.test/hooks/${id}` }, + }), + ), + ]; + const result = await reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + dryRun: true, + fetchFn: fakeFetch(requests, responses), + }); + expect(result).toEqual({ actionId: null, operation: "create" }); + expect(requests[1]!.url).toContain("start-index=0"); + expect(requests[2]!.url).toContain("start-index=100"); + expect(requests).toHaveLength(103); + }); + + test("creates a missing action without exposing credentials", async () => { + const requests: Request[] = []; + const result = await reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + fetchFn: fakeFetch(requests, [ + Response.json(SOURCE), + Response.json([]), + Response.json({ id: "71" }), + Response.json({ ...desiredAlertAction(SOURCE, WEBHOOK_SECRET), id: "71" }), + ]), + }); + expect(result).toEqual({ actionId: "71", operation: "create" }); + expect(requests.map((request) => request.method)).toEqual([ + "GET", + "GET", + "POST", + "GET", + ]); + expect(requests[2]!.headers.get("authorization")).toBe(API_KEY); + }); + + test("updates one drifted action and leaves an equivalent action unchanged", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET); + const drifted = { ...desired, id: "72", triggerMode: "MANUAL" }; + const updateRequests: Request[] = []; + const updated = await reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + fetchFn: fakeFetch(updateRequests, [ + Response.json(SOURCE), + Response.json([{ id: "72" }]), + Response.json(drifted), + Response.json({ ...desired, id: "action-72" }), + Response.json({ ...desired, id: "action-72" }), + ]), + }); + expect(updated).toEqual({ actionId: "action-72", operation: "update" }); + expect(updateRequests[3]!.method).toBe("PUT"); + + const unchangedRequests: Request[] = []; + const unchanged = await reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + fetchFn: fakeFetch(unchangedRequests, [ + Response.json(SOURCE), + Response.json([{ id: "72" }]), + Response.json({ ...desired, id: "72" }), + ]), + }); + expect(unchanged).toEqual({ actionId: "72", operation: "unchanged" }); + expect(unchangedRequests).toHaveLength(3); + }); + + test("fails closed on duplicate candidates and does not delete either", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET); + const requests: Request[] = []; + await expect( + reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + fetchFn: fakeFetch(requests, [ + Response.json(SOURCE), + Response.json([ + { id: "72" }, + { id: "73" }, + ]), + Response.json({ ...desired, id: "72" }), + Response.json({ ...desired, id: "73" }), + ]), + }), + ).rejects.toThrow("Multiple Postil webhook alert actions exist"); + expect(requests).toHaveLength(4); + }); + + test("reports only an HTTP status when the provider rejects a request", async () => { + const providerBody = "provider response containing credential material"; + await expect( + reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + fetchFn: async () => new Response(providerBody, { status: 403 }), + }), + ).rejects.toThrow("HTTP 403"); + try { + await reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + fetchFn: async () => new Response(providerBody, { status: 403 }), + }); + } catch (error) { + expect(String(error)).not.toContain(providerBody); + } + }); + + test("proves webhook action success and resolves its unique canary", async () => { + const requests: Request[] = []; + const responses = [ + new Response(null, { status: 202 }), + new Response(null, { status: 202 }), + ]; + let canaryKey = ""; + const fetchFn: Fetch = async (input, init) => { + const request = new Request(input, init); + requests.push(request); + if (request.method === "POST" && request.url.endsWith("/events")) { + const body = (await request.clone().json()) as Record; + if (body.eventType === "ALERT") canaryKey = String(body.alertKey); + return responses.shift()!; + } + if (request.url.includes("/alerts?")) { + const resolveSent = requests.filter( + (value) => value.method === "POST" && value.url.endsWith("/events"), + ).length > 1; + return Response.json([ + { + id: 99, + alertKey: canaryKey, + status: resolveSent ? "RESOLVED" : "PENDING", + }, + ]); + } + if (request.url.endsWith("/alerts/99/actions")) { + const resolveSent = requests.filter( + (value) => value.method === "POST" && value.url.endsWith("/events"), + ).length > 1; + return Response.json({ + alertActionId: 72, + history: resolveSent + ? [{ success: true }, { success: true }] + : [{ success: true }], + }); + } + return responses.shift()!; + }; + + await verifyIlertAlertStreamCanary({ + actionId: "72", + apiKey: API_KEY, + integrationKey: "test-integration-key", + fetchFn, + sleep: async () => undefined, + runId: "100", + runAttempt: "2", + }); + const eventBodies = await Promise.all( + requests + .filter((request) => request.method === "POST") + .map((request) => request.clone().json() as Promise>), + ); + expect(eventBodies.map((body) => body.eventType)).toEqual(["ALERT", "RESOLVE"]); + expect(eventBodies[0]!.alertKey).toBe(eventBodies[1]!.alertKey); + }); +}); + +function fakeFetch(requests: Request[], responses: Response[]): Fetch { + return async (input, init) => { + requests.push(new Request(input, init)); + const response = responses.shift(); + if (!response) throw new Error("unexpected request"); + return response; + }; +}