From a2518d14915eadbb938de34751e0c2c23f80a69d Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 16:14:25 +0000 Subject: [PATCH 1/6] Reconcile the iLert operator alert stream --- .env.example | 10 +- .github/workflows/production-monitor.yml | 73 ++- scripts/reconcile-ilert-alert-stream.ts | 581 +++++++++++++++++++++ tests/health.test.ts | 31 ++ tests/reconcile-ilert-alert-stream.test.ts | 281 ++++++++++ 5 files changed, 959 insertions(+), 17 deletions(-) create mode 100644 scripts/reconcile-ilert-alert-stream.ts create mode 100644 tests/reconcile-ilert-alert-stream.test.ts diff --git a/.env.example b/.env.example index e50751c1..fe0d3fbc 100644 --- a/.env.example +++ b/.env.example @@ -27,11 +27,11 @@ POSTIL_SESSION_SECRET= GITHUB_WEBHOOK_SECRET= # Optional iLert outbound alert receiver. Generate a random password with -# `openssl rand -hex 32`, set it here, and configure the iLert webhook URL as -# https://postil-ilert:@/api/webhooks/ilert. -# Select alert-created, alert-assigned, alert-auto-escalated, -# alert-auto-resolved, alert-acknowledged, alert-rejected, alert-raised, and -# alert-resolved as trigger events. Leave unset to make the endpoint return 404. +# `openssl rand -hex 32`, set it here, and configure a Webhook alert action for +# `https:///api/webhooks/ilert`. Send an `Authorization` header with +# HTTP Basic credentials using `postil-ilert` as the username and this value as +# the password. Select alert-created, alert-acknowledged, alert-comment-added, +# and alert-resolved as trigger events. Leave unset to return 404. POSTIL_ILERT_WEBHOOK_SECRET= # Numeric GitHub ids allowed to use operator-only surfaces. The alert stream diff --git a/.github/workflows/production-monitor.yml b/.github/workflows/production-monitor.yml index b50c43c5..b0f0c8b8 100644 --- a/.github/workflows/production-monitor.yml +++ b/.github/workflows/production-monitor.yml @@ -389,6 +389,59 @@ jobs: fi fi + alert-stream: + name: Verify operator alert stream + needs: smoke + if: ${{ inputs.test_alert == true }} + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.3.14 + - name: Load iLert management secret from Infisical + uses: Infisical/secrets-action@77ab1f4ccd183a543cb5b42435fbd181189f4995 # v1.0.16 + with: + method: oidc + identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} + project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }} + env-slug: prod + domain: https://eu.infisical.com + secret-path: / + secret-name: ILERT_API_KEY + - name: Load iLert integration secret from Infisical + uses: Infisical/secrets-action@77ab1f4ccd183a543cb5b42435fbd181189f4995 # v1.0.16 + with: + method: oidc + identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} + project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }} + env-slug: prod + domain: https://eu.infisical.com + secret-path: /postil + secret-name: ILERT_INTEGRATION_KEY + - name: Load Postil webhook secret from Infisical + uses: Infisical/secrets-action@77ab1f4ccd183a543cb5b42435fbd181189f4995 # v1.0.16 + with: + method: oidc + identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} + project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }} + env-slug: prod + domain: https://eu.infisical.com + secret-path: /postil + secret-name: POSTIL_ILERT_WEBHOOK_SECRET + - name: Preview iLert alert-stream reconciliation + env: + POSTIL_ILERT_ALERT_SOURCE_ID: ${{ vars.POSTIL_ILERT_ALERT_SOURCE_ID }} + run: bun run scripts/reconcile-ilert-alert-stream.ts --dry-run + - name: Reconcile and verify the iLert alert stream + env: + POSTIL_ILERT_ALERT_SOURCE_ID: ${{ vars.POSTIL_ILERT_ALERT_SOURCE_ID }} + run: bun run scripts/reconcile-ilert-alert-stream.ts --canary + # Alert delivery is owned by the external alerting service (ilert). The # platform and this workflow only report events; paging, escalation, and # deduplication happen there. A stable alertKey groups consecutive failing @@ -396,7 +449,7 @@ jobs: notify: name: Raise external alert needs: smoke - if: ${{ always() && (needs.smoke.result == 'failure' || inputs.test_alert == true) }} + if: ${{ always() && needs.smoke.result == 'failure' }} permissions: contents: read id-token: write @@ -417,23 +470,19 @@ jobs: uses: ./.github/actions/ilert-event with: event-type: ALERT - summary: ${{ needs.smoke.result == 'failure' && 'Postil production monitor failed' || 'Postil production monitor test alert' }} - alert-key: ${{ needs.smoke.result == 'failure' && 'postil-production-monitor' || 'postil-production-monitor-test' }} + summary: Postil production monitor failed + alert-key: postil-production-monitor details: >- - ${{ needs.smoke.result == 'failure' - && 'Production checks failed. Run log:' - || 'Operator-requested test alert; production checks passed. Run log:' }} + Production checks failed. Run log: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} - # A production failure pages if it can and records the gap if it - # cannot, because the failing check is already the signal. A test - # alert exists only to prove delivery works, so an undelivered one is - # the failure it was run to detect. - require-delivery: ${{ inputs.test_alert == true }} + # The production check is already the signal, so record an alerting + # delivery gap without replacing the original failure. + require-delivery: false resolve: name: Resolve external alert needs: smoke - if: ${{ needs.smoke.result == 'success' && inputs.test_alert != true }} + if: ${{ needs.smoke.result == 'success' }} permissions: contents: read id-token: write diff --git a/scripts/reconcile-ilert-alert-stream.ts b/scripts/reconcile-ilert-alert-stream.ts new file mode 100644 index 00000000..3eb0c7fa --- /dev/null +++ b/scripts/reconcile-ilert-alert-stream.ts @@ -0,0 +1,581 @@ +#!/usr/bin/env bun + +import { randomUUID } from "node:crypto"; + +const API_BASE = "https://api.ilert.com/api"; +const WEBHOOK_URL = "https://postil.dev/api/webhooks/ilert"; +const WEBHOOK_USERNAME = "postil-ilert"; +const ACTION_NAME = "Postil operator alert stream"; +const REQUEST_TIMEOUT_MS = 20_000; +const CANARY_ATTEMPTS = 12; +const CANARY_RETRY_MS = 5_000; + +export const ALERT_TRIGGER_TYPES = [ + "alert-created", + "alert-acknowledged", + "alert-comment-added", + "alert-resolved", +] as const; + +export type Fetch = ( + input: string | URL | Request, + init?: RequestInit, +) => Promise; +type Sleep = (milliseconds: number) => Promise; + +interface IlertAlertAction { + id?: unknown; + name?: unknown; + connectorType?: unknown; + triggerMode?: unknown; + triggerTypes?: unknown; + alertSources?: unknown; + params?: unknown; +} + +interface IlertAlertSource { + id: number; + name: string; + integrationType: string; + escalationPolicy: { + id?: number; + name: string; + escalationRules: unknown[]; + }; +} + +interface ReconcileOptions { + apiKey: string; + sourceId: number; + webhookSecret: string; + dryRun?: boolean; + fetchFn?: Fetch; +} + +interface CanaryOptions { + actionId: string; + apiKey: string; + integrationKey: string; + fetchFn?: Fetch; + sleep?: Sleep; + runId?: string; + runAttempt?: string; +} + +export type ReconcileOperation = "create" | "update" | "unchanged"; + +export interface ReconcileResult { + actionId: string | null; + operation: ReconcileOperation; +} + +export function desiredAlertAction( + source: IlertAlertSource, + webhookSecret: string, +): Record { + const authorization = Buffer.from( + `${WEBHOOK_USERNAME}:${webhookSecret}`, + "utf8", + ).toString("base64"); + return { + alertSources: [source], + connectorType: "webhook", + name: ACTION_NAME, + triggerMode: "AUTOMATIC", + triggerTypes: [...ALERT_TRIGGER_TYPES], + params: { + webhookUrl: WEBHOOK_URL, + headers: [{ key: "Authorization", value: `Basic ${authorization}` }], + }, + }; +} + +export function equivalentAlertAction( + actual: IlertAlertAction, + desired: Record, +): boolean { + const actualSources = relationIds(actual.alertSources); + const desiredSources = relationIds(desired.alertSources); + const actualParams = record(actual.params); + const desiredParams = record(desired.params); + return ( + actual.name === desired.name && + actual.connectorType === desired.connectorType && + actual.triggerMode === desired.triggerMode && + sameStringSet(actual.triggerTypes, desired.triggerTypes) && + sameNumberSet(actualSources, desiredSources) && + actualParams?.webhookUrl === desiredParams?.webhookUrl && + sameHeaders(actualParams?.headers, desiredParams?.headers) + ); +} + +export async function reconcileIlertAlertAction( + options: ReconcileOptions, +): Promise { + const fetchFn = options.fetchFn ?? fetch; + const source = alertSource( + await ilertJson( + fetchFn, + options.apiKey, + `/alert-sources/${options.sourceId}`, + ), + options.sourceId, + ); + const desired = desiredAlertAction(source, options.webhookSecret); + const listedActions = await listIlertAlertActions( + fetchFn, + options.apiKey, + ); + const actions: unknown[] = []; + for (let index = 0; index < listedActions.length; index += 8) { + actions.push( + ...(await Promise.all( + listedActions.slice(index, index + 8).map((action) => + ilertJson( + fetchFn, + options.apiKey, + `/alert-actions/${encodeURIComponent(requiredActionId(action))}`, + ), + ), + )), + ); + } + + const candidates = actions.filter((value): value is IlertAlertAction => { + const action = record(value); + if (!action) return false; + const params = record(action.params); + return action.name === ACTION_NAME || params?.webhookUrl === WEBHOOK_URL; + }); + for (const candidate of candidates) { + if ( + candidate.connectorType !== "webhook" || + !sameNumberSet(relationIds(candidate.alertSources), [options.sourceId]) + ) { + throw new Error( + "A conflicting Postil alert action exists; refusing to change its type or source scope", + ); + } + } + if (candidates.length > 1) { + throw new Error( + "Multiple Postil webhook alert actions exist; refusing to choose or delete one", + ); + } + + const existing = candidates[0]; + if (!existing) { + if (options.dryRun) return { actionId: null, operation: "create" }; + const created = await ilertJson(fetchFn, options.apiKey, "/alert-actions", { + method: "POST", + body: JSON.stringify(desired), + }); + const actionId = requiredActionId(created); + await verifyReconciledAction(fetchFn, options.apiKey, actionId, desired); + return { actionId, operation: "create" }; + } + + const actionId = requiredActionId(existing); + if (equivalentAlertAction(existing, desired)) { + return { actionId, operation: "unchanged" }; + } + if (options.dryRun) return { actionId, operation: "update" }; + const updated = await ilertJson( + fetchFn, + options.apiKey, + `/alert-actions/${encodeURIComponent(actionId)}`, + { method: "PUT", body: JSON.stringify({ ...desired, id: actionId }) }, + ); + const updatedId = requiredActionId(updated); + await verifyReconciledAction(fetchFn, options.apiKey, updatedId, desired); + return { actionId: updatedId, operation: "update" }; +} + +export async function verifyIlertAlertStreamCanary( + options: CanaryOptions, +): Promise { + const fetchFn = options.fetchFn ?? fetch; + const sleep = options.sleep ?? Bun.sleep; + const startedAt = new Date(Date.now() - 5_000).toISOString(); + const canaryKey = [ + "postil-operator-alert-stream-canary", + options.runId ?? "local", + options.runAttempt ?? "1", + randomUUID(), + ].join("-"); + let alertAccepted = false; + let resolveSent = false; + let primaryError: unknown; + + try { + await postIlertEvent(fetchFn, { + integrationKey: options.integrationKey, + eventType: "ALERT", + summary: "Postil operator alert stream canary", + details: "GitHub Actions is verifying the Postil operator notification path.", + alertKey: canaryKey, + priority: "HIGH", + }); + alertAccepted = true; + + const created = await waitForCanaryDelivery({ + actionId: options.actionId, + apiKey: options.apiKey, + canaryKey, + fetchFn, + sleep, + startedAt, + }); + await postIlertEvent(fetchFn, { + integrationKey: options.integrationKey, + eventType: "RESOLVE", + summary: "Postil operator alert stream canary resolved", + alertKey: canaryKey, + }); + resolveSent = true; + await waitForCanaryDelivery({ + actionId: options.actionId, + alertId: created.alertId, + apiKey: options.apiKey, + canaryKey, + fetchFn, + minimumSuccessfulDeliveries: created.successfulDeliveries + 1, + requiredStatus: "RESOLVED", + sleep, + startedAt, + }); + } catch (error) { + primaryError = error; + throw error; + } finally { + if (alertAccepted && !resolveSent) { + try { + await postIlertEvent(fetchFn, { + integrationKey: options.integrationKey, + eventType: "RESOLVE", + summary: "Postil operator alert stream canary resolved", + alertKey: canaryKey, + }); + } catch (resolveError) { + if (primaryError === undefined) throw resolveError; + } + } + } +} + +interface WaitForCanaryOptions { + actionId: string; + alertId?: string; + apiKey: string; + canaryKey: string; + fetchFn: Fetch; + minimumSuccessfulDeliveries?: number; + requiredStatus?: string; + sleep: Sleep; + startedAt: string; +} + +async function waitForCanaryDelivery( + options: WaitForCanaryOptions, +): Promise<{ alertId: string; successfulDeliveries: number }> { + for (let attempt = 0; attempt < CANARY_ATTEMPTS; attempt += 1) { + const alerts = await ilertJson( + options.fetchFn, + options.apiKey, + `/alerts?from=${encodeURIComponent(options.startedAt)}&max-results=100`, + ); + if (!Array.isArray(alerts)) { + throw new Error("iLert returned an invalid alert list during the canary"); + } + const alert = alerts.find( + (value) => record(value)?.alertKey === options.canaryKey, + ); + const alertRecord = record(alert); + const alertId = positiveAlertId(alertRecord?.id); + if (alertId && (!options.alertId || options.alertId === alertId)) { + const actions = await ilertJson( + options.fetchFn, + options.apiKey, + `/alerts/${encodeURIComponent(alertId)}/actions`, + ); + const successfulDeliveries = successfulActionDeliveries( + actions, + options.actionId, + ); + if ( + successfulDeliveries >= (options.minimumSuccessfulDeliveries ?? 1) && + (!options.requiredStatus || alertRecord?.status === options.requiredStatus) + ) { + return { alertId, successfulDeliveries }; + } + } + if (attempt + 1 < CANARY_ATTEMPTS) await options.sleep(CANARY_RETRY_MS); + } + throw new Error("iLert did not confirm successful Postil webhook delivery"); +} + +function successfulActionDeliveries(value: unknown, actionId: string): number { + const actions = Array.isArray(value) ? value : [value]; + if (actions.some((action) => !record(action))) { + throw new Error("iLert returned invalid action history during the canary"); + } + return actions + .filter((action) => record(action)?.alertActionId === actionId) + .flatMap((action) => { + const history = record(action)?.history; + if (!Array.isArray(history)) return []; + return history; + }) + .filter((entry) => record(entry)?.success === true).length; +} + +async function postIlertEvent( + fetchFn: Fetch, + payload: Record, +): Promise { + const response = await fetchWithTimeout(fetchFn, `${API_BASE}/events`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(payload), + }); + if (!response.ok) { + throw new Error(`iLert event request failed with HTTP ${response.status}`); + } +} + +async function ilertJson( + fetchFn: Fetch, + apiKey: string, + path: string, + init: RequestInit = {}, +): Promise { + const response = await fetchWithTimeout(fetchFn, `${API_BASE}${path}`, { + ...init, + headers: { + accept: "application/json", + authorization: apiKey, + ...(init.body ? { "content-type": "application/json" } : {}), + }, + }); + if (!response.ok) { + throw new Error(`iLert management request failed with HTTP ${response.status}`); + } + try { + return await response.json(); + } catch { + throw new Error("iLert returned invalid JSON"); + } +} + +async function listIlertAlertActions( + fetchFn: Fetch, + apiKey: string, +): Promise { + const actions: IlertAlertAction[] = []; + for (let startIndex = 0; startIndex <= 1_000; startIndex += 100) { + const page = await ilertJson( + fetchFn, + apiKey, + `/alert-actions?start-index=${startIndex}&max-results=100`, + ); + if (!Array.isArray(page) || page.some((value) => !record(value))) { + throw new Error("iLert returned an invalid alert-action list"); + } + actions.push(...(page as IlertAlertAction[])); + if (page.length < 100) return actions; + } + throw new Error("iLert alert-action pagination exceeded the safety bound"); +} + +async function verifyReconciledAction( + fetchFn: Fetch, + apiKey: string, + actionId: string, + desired: Record, +): Promise { + const confirmed = await ilertJson( + fetchFn, + apiKey, + `/alert-actions/${encodeURIComponent(actionId)}`, + ); + const action = record(confirmed); + if (!action || !equivalentAlertAction(action, desired)) { + throw new Error("iLert did not retain the reconciled alert action"); + } +} + +async function fetchWithTimeout( + fetchFn: Fetch, + url: string, + init: RequestInit, +): Promise { + return fetchFn(url, { + ...init, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); +} + +function requiredActionId(value: unknown): string { + const id = opaqueActionId(record(value)?.id); + if (!id) throw new Error("iLert returned an alert action without an identity"); + return id; +} + +function alertSource(value: unknown, expectedId: number): IlertAlertSource { + const source = record(value); + const escalationPolicy = record(source?.escalationPolicy); + const id = positiveSafeNumber(source?.id); + const name = source?.name; + const integrationType = source?.integrationType; + const policyName = escalationPolicy?.name; + const escalationRules = escalationPolicy?.escalationRules; + if ( + id !== expectedId || + typeof name !== "string" || + name.length === 0 || + typeof integrationType !== "string" || + integrationType.length === 0 || + typeof policyName !== "string" || + policyName.length === 0 || + !Array.isArray(escalationRules) + ) { + throw new Error("iLert returned an invalid alert source"); + } + const policyId = positiveSafeNumber(escalationPolicy?.id); + return { + id, + name, + integrationType, + escalationPolicy: { + ...(policyId === null ? {} : { id: policyId }), + name: policyName, + escalationRules, + }, + }; +} + +function positiveSafeNumber(value: unknown): number | null { + if (typeof value === "number" && Number.isSafeInteger(value) && value > 0) { + return value; + } + if (typeof value === "string" && /^[1-9][0-9]*$/u.test(value)) { + const parsed = Number(value); + if (Number.isSafeInteger(parsed)) return parsed; + } + return null; +} + +function opaqueActionId(value: unknown): string | null { + if ( + typeof value === "string" && + value.length <= 128 && + /^[A-Za-z0-9._:-]+$/u.test(value) + ) { + return value; + } + if (typeof value === "number" && Number.isSafeInteger(value) && value > 0) { + return String(value); + } + return null; +} + +function positiveAlertId(value: unknown): string | null { + if (typeof value === "number" && Number.isSafeInteger(value) && value > 0) { + return String(value); + } + if (typeof value === "string" && /^[1-9][0-9]*$/u.test(value)) return value; + return null; +} + +function record(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : null; +} + +function relationIds(value: unknown): number[] { + if (!Array.isArray(value)) return []; + return value.flatMap((entry) => { + const id = record(entry)?.id; + const parsed = positiveSafeNumber(id); + return parsed === null ? [] : [parsed]; + }); +} + +function sameStringSet(left: unknown, right: unknown): boolean { + if (!Array.isArray(left) || !Array.isArray(right)) return false; + if (!left.every((value) => typeof value === "string")) return false; + if (!right.every((value) => typeof value === "string")) return false; + return [...left].sort().join("\0") === [...right].sort().join("\0"); +} + +function sameNumberSet(left: number[], right: number[]): boolean { + return [...left].sort((a, b) => a - b).join("\0") === + [...right].sort((a, b) => a - b).join("\0"); +} + +function sameHeaders(left: unknown, right: unknown): boolean { + if (!Array.isArray(left) || !Array.isArray(right) || left.length !== right.length) { + return false; + } + const normalize = (values: unknown[]) => + values + .map((value) => record(value)) + .filter((value): value is Record => value !== null) + .map((value) => `${String(value.key).toLowerCase()}:${String(value.value)}`) + .sort(); + return normalize(left).join("\0") === normalize(right).join("\0"); +} + +function requiredEnvironment(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} + +function sourceIdFromEnvironment(): number { + const raw = requiredEnvironment("POSTIL_ILERT_ALERT_SOURCE_ID"); + const value = Number(raw); + if (!Number.isSafeInteger(value) || value <= 0 || String(value) !== raw) { + throw new Error("POSTIL_ILERT_ALERT_SOURCE_ID must be a positive integer"); + } + return value; +} + +async function main(): Promise { + const allowed = new Set(["--dry-run", "--canary"]); + if (process.argv.slice(2).some((argument) => !allowed.has(argument))) { + throw new Error("usage: reconcile-ilert-alert-stream.ts [--dry-run] [--canary]"); + } + const dryRun = process.argv.includes("--dry-run"); + const canary = process.argv.includes("--canary"); + if (dryRun && canary) throw new Error("--canary cannot be combined with --dry-run"); + + const apiKey = requiredEnvironment("ILERT_API_KEY"); + const result = await reconcileIlertAlertAction({ + apiKey, + sourceId: sourceIdFromEnvironment(), + webhookSecret: requiredEnvironment("POSTIL_ILERT_WEBHOOK_SECRET"), + dryRun, + }); + console.log( + dryRun + ? `iLert alert-stream reconciliation plan: ${result.operation}` + : `iLert alert-stream reconciliation: ${result.operation}`, + ); + if (canary) { + if (!result.actionId) throw new Error("the reconciled alert action has no identity"); + await verifyIlertAlertStreamCanary({ + actionId: result.actionId, + apiKey, + integrationKey: requiredEnvironment("ILERT_INTEGRATION_KEY"), + runId: process.env.GITHUB_RUN_ID, + runAttempt: process.env.GITHUB_RUN_ATTEMPT, + }); + console.log("iLert confirmed successful Postil webhook delivery"); + } +} + +if (import.meta.main) { + await main(); +} diff --git a/tests/health.test.ts b/tests/health.test.ts index ef0a6c86..a04535ba 100644 --- a/tests/health.test.ts +++ b/tests/health.test.ts @@ -158,6 +158,37 @@ describe("/api/health/monitor", () => { }); describe("production monitor workflow", () => { + test("owns a bounded manual alert-stream canary and resolves test alerts", async () => { + const source = await readFile( + new URL("../.github/workflows/production-monitor.yml", import.meta.url), + "utf8", + ); + const workflow = parse(source) as { + jobs: Record< + string, + { + if?: string; + needs?: string; + steps?: Array<{ name?: string; run?: string }>; + } + >; + }; + const alertStream = workflow.jobs["alert-stream"]; + expect(alertStream?.needs).toBe("smoke"); + expect(alertStream?.if).toContain("inputs.test_alert == true"); + expect(alertStream?.steps?.map((step) => step.name)).toContain( + "Preview iLert alert-stream reconciliation", + ); + expect(alertStream?.steps?.map((step) => step.name)).toContain( + "Reconcile and verify the iLert alert stream", + ); + expect(workflow.jobs.notify?.if).not.toContain("inputs.test_alert"); + expect(workflow.jobs.resolve?.if).toContain( + "needs.smoke.result == 'success'", + ); + expect(workflow.jobs.resolve?.if).not.toContain("inputs.test_alert"); + }); + test("enforces monitor health, collection, delivery, failure, and stuck-pass signals", async () => { const source = await readFile( new URL("../.github/workflows/production-monitor.yml", import.meta.url), diff --git a/tests/reconcile-ilert-alert-stream.test.ts b/tests/reconcile-ilert-alert-stream.test.ts new file mode 100644 index 00000000..0a356ce4 --- /dev/null +++ b/tests/reconcile-ilert-alert-stream.test.ts @@ -0,0 +1,281 @@ +import { describe, expect, test } from "bun:test"; + +import { + ALERT_TRIGGER_TYPES, + desiredAlertAction, + equivalentAlertAction, + type Fetch, + reconcileIlertAlertAction, + verifyIlertAlertStreamCanary, +} from "../scripts/reconcile-ilert-alert-stream"; + +const API_KEY = "test-api-key"; +const SOURCE_ID = 42; +const WEBHOOK_SECRET = "test-webhook-secret-with-at-least-32-bytes"; +const SOURCE = { + id: SOURCE_ID, + name: "Postil test source", + integrationType: "API", + escalationPolicy: { + id: 7, + name: "Test escalation", + escalationRules: [], + }, +}; + +describe("iLert alert-stream reconciliation", () => { + test("builds the bounded automatic webhook action", () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET); + expect(desired).toMatchObject({ + alertSources: [SOURCE], + connectorType: "webhook", + name: "Postil operator alert stream", + triggerMode: "AUTOMATIC", + triggerTypes: [...ALERT_TRIGGER_TYPES], + params: { + webhookUrl: "https://postil.dev/api/webhooks/ilert", + headers: [{ key: "Authorization" }], + }, + }); + expect(JSON.stringify(desired)).not.toContain(WEBHOOK_SECRET); + }); + + test("compares trigger and header sets without depending on order or case", () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET); + const actual = structuredClone(desired) as Record; + actual.id = "42"; + actual.triggerTypes = [...ALERT_TRIGGER_TYPES].reverse(); + const params = actual.params as { headers: Array<{ key: string; value: string }> }; + params.headers[0]!.key = "authorization"; + expect(equivalentAlertAction(actual, desired)).toBe(true); + }); + + test("dry-run reports create without mutating", async () => { + const requests: Request[] = []; + const result = await reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + dryRun: true, + fetchFn: fakeFetch(requests, [Response.json(SOURCE), Response.json([])]), + }); + expect(result).toEqual({ actionId: null, operation: "create" }); + expect(requests).toHaveLength(2); + expect(requests[0]!.method).toBe("GET"); + }); + + test("rejects an incomplete source instead of sending a lossy relation", async () => { + await expect( + reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + dryRun: true, + fetchFn: fakeFetch([], [ + Response.json({ id: SOURCE_ID, name: SOURCE.name }), + ]), + }), + ).rejects.toThrow("invalid alert source"); + }); + + test("checks every action page before planning a create", async () => { + const requests: Request[] = []; + const firstPage = Array.from({ length: 100 }, (_, index) => ({ + id: String(index + 1), + })); + const responses = [ + Response.json(SOURCE), + Response.json(firstPage), + Response.json([]), + ...firstPage.map(({ id }) => + Response.json({ + id, + name: `Unrelated action ${id}`, + connectorType: "webhook", + alertSources: [{ id: SOURCE_ID + 1 }], + params: { webhookUrl: `https://example.test/hooks/${id}` }, + }), + ), + ]; + const result = await reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + dryRun: true, + fetchFn: fakeFetch(requests, responses), + }); + expect(result).toEqual({ actionId: null, operation: "create" }); + expect(requests[1]!.url).toContain("start-index=0"); + expect(requests[2]!.url).toContain("start-index=100"); + expect(requests).toHaveLength(103); + }); + + test("creates a missing action without exposing credentials", async () => { + const requests: Request[] = []; + const result = await reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + fetchFn: fakeFetch(requests, [ + Response.json(SOURCE), + Response.json([]), + Response.json({ id: "71" }), + Response.json({ ...desiredAlertAction(SOURCE, WEBHOOK_SECRET), id: "71" }), + ]), + }); + expect(result).toEqual({ actionId: "71", operation: "create" }); + expect(requests.map((request) => request.method)).toEqual([ + "GET", + "GET", + "POST", + "GET", + ]); + expect(requests[2]!.headers.get("authorization")).toBe(API_KEY); + }); + + test("updates one drifted action and leaves an equivalent action unchanged", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET); + const drifted = { ...desired, id: "72", triggerMode: "MANUAL" }; + const updateRequests: Request[] = []; + const updated = await reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + fetchFn: fakeFetch(updateRequests, [ + Response.json(SOURCE), + Response.json([{ id: "72" }]), + Response.json(drifted), + Response.json({ ...desired, id: "action-72" }), + Response.json({ ...desired, id: "action-72" }), + ]), + }); + expect(updated).toEqual({ actionId: "action-72", operation: "update" }); + expect(updateRequests[3]!.method).toBe("PUT"); + + const unchangedRequests: Request[] = []; + const unchanged = await reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + fetchFn: fakeFetch(unchangedRequests, [ + Response.json(SOURCE), + Response.json([{ id: "72" }]), + Response.json({ ...desired, id: "72" }), + ]), + }); + expect(unchanged).toEqual({ actionId: "72", operation: "unchanged" }); + expect(unchangedRequests).toHaveLength(3); + }); + + test("fails closed on duplicate candidates and does not delete either", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET); + const requests: Request[] = []; + await expect( + reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + fetchFn: fakeFetch(requests, [ + Response.json(SOURCE), + Response.json([ + { id: "72" }, + { id: "73" }, + ]), + Response.json({ ...desired, id: "72" }), + Response.json({ ...desired, id: "73" }), + ]), + }), + ).rejects.toThrow("Multiple Postil webhook alert actions exist"); + expect(requests).toHaveLength(4); + }); + + test("reports only an HTTP status when the provider rejects a request", async () => { + const providerBody = "provider response containing credential material"; + await expect( + reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + fetchFn: async () => new Response(providerBody, { status: 403 }), + }), + ).rejects.toThrow("HTTP 403"); + try { + await reconcileIlertAlertAction({ + apiKey: API_KEY, + sourceId: SOURCE_ID, + webhookSecret: WEBHOOK_SECRET, + fetchFn: async () => new Response(providerBody, { status: 403 }), + }); + } catch (error) { + expect(String(error)).not.toContain(providerBody); + } + }); + + test("proves webhook action success and resolves its unique canary", async () => { + const requests: Request[] = []; + const responses = [ + new Response(null, { status: 202 }), + new Response(null, { status: 202 }), + ]; + let canaryKey = ""; + const fetchFn: Fetch = async (input, init) => { + const request = new Request(input, init); + requests.push(request); + if (request.method === "POST" && request.url.endsWith("/events")) { + const body = (await request.clone().json()) as Record; + if (body.eventType === "ALERT") canaryKey = String(body.alertKey); + return responses.shift()!; + } + if (request.url.includes("/alerts?")) { + const resolveSent = requests.filter( + (value) => value.method === "POST" && value.url.endsWith("/events"), + ).length > 1; + return Response.json([ + { + id: 99, + alertKey: canaryKey, + status: resolveSent ? "RESOLVED" : "PENDING", + }, + ]); + } + if (request.url.endsWith("/alerts/99/actions")) { + const resolveSent = requests.filter( + (value) => value.method === "POST" && value.url.endsWith("/events"), + ).length > 1; + return Response.json({ + alertActionId: "72", + history: resolveSent + ? [{ success: true }, { success: true }] + : [{ success: true }], + }); + } + return responses.shift()!; + }; + + await verifyIlertAlertStreamCanary({ + actionId: "72", + apiKey: API_KEY, + integrationKey: "test-integration-key", + fetchFn, + sleep: async () => undefined, + runId: "100", + runAttempt: "2", + }); + const eventBodies = await Promise.all( + requests + .filter((request) => request.method === "POST") + .map((request) => request.clone().json() as Promise>), + ); + expect(eventBodies.map((body) => body.eventType)).toEqual(["ALERT", "RESOLVE"]); + expect(eventBodies[0]!.alertKey).toBe(eventBodies[1]!.alertKey); + }); +}); + +function fakeFetch(requests: Request[], responses: Response[]): Fetch { + return async (input, init) => { + requests.push(new Request(input, init)); + const response = responses.shift(); + if (!response) throw new Error("unexpected request"); + return response; + }; +} From 581dbe1cb6ac6650e69c8711c9ad04e121f0492b Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 16:28:33 +0000 Subject: [PATCH 2/6] Simplify iLert alert reconciliation --- scripts/reconcile-ilert-alert-stream.ts | 558 ++++++++++-------------- 1 file changed, 235 insertions(+), 323 deletions(-) diff --git a/scripts/reconcile-ilert-alert-stream.ts b/scripts/reconcile-ilert-alert-stream.ts index 3eb0c7fa..6b522365 100644 --- a/scripts/reconcile-ilert-alert-stream.ts +++ b/scripts/reconcile-ilert-alert-stream.ts @@ -4,7 +4,6 @@ import { randomUUID } from "node:crypto"; const API_BASE = "https://api.ilert.com/api"; const WEBHOOK_URL = "https://postil.dev/api/webhooks/ilert"; -const WEBHOOK_USERNAME = "postil-ilert"; const ACTION_NAME = "Postil operator alert stream"; const REQUEST_TIMEOUT_MS = 20_000; const CANARY_ATTEMPTS = 12; @@ -17,32 +16,13 @@ export const ALERT_TRIGGER_TYPES = [ "alert-resolved", ] as const; +type Json = Record; export type Fetch = ( input: string | URL | Request, init?: RequestInit, ) => Promise; type Sleep = (milliseconds: number) => Promise; - -interface IlertAlertAction { - id?: unknown; - name?: unknown; - connectorType?: unknown; - triggerMode?: unknown; - triggerTypes?: unknown; - alertSources?: unknown; - params?: unknown; -} - -interface IlertAlertSource { - id: number; - name: string; - integrationType: string; - escalationPolicy: { - id?: number; - name: string; - escalationRules: unknown[]; - }; -} +type Operation = "create" | "update" | "unchanged"; interface ReconcileOptions { apiKey: string; @@ -62,21 +42,15 @@ interface CanaryOptions { runAttempt?: string; } -export type ReconcileOperation = "create" | "update" | "unchanged"; - export interface ReconcileResult { actionId: string | null; - operation: ReconcileOperation; + operation: Operation; } -export function desiredAlertAction( - source: IlertAlertSource, - webhookSecret: string, -): Record { - const authorization = Buffer.from( - `${WEBHOOK_USERNAME}:${webhookSecret}`, - "utf8", - ).toString("base64"); +export function desiredAlertAction(source: Json, secret: string): Json { + const authorization = Buffer.from(`postil-ilert:${secret}`, "utf8").toString( + "base64", + ); return { alertSources: [source], connectorType: "webhook", @@ -90,22 +64,17 @@ export function desiredAlertAction( }; } -export function equivalentAlertAction( - actual: IlertAlertAction, - desired: Record, -): boolean { - const actualSources = relationIds(actual.alertSources); - const desiredSources = relationIds(desired.alertSources); - const actualParams = record(actual.params); - const desiredParams = record(desired.params); +export function equivalentAlertAction(actual: Json, desired: Json): boolean { + const left = object(actual.params); + const right = object(desired.params); return ( actual.name === desired.name && actual.connectorType === desired.connectorType && actual.triggerMode === desired.triggerMode && - sameStringSet(actual.triggerTypes, desired.triggerTypes) && - sameNumberSet(actualSources, desiredSources) && - actualParams?.webhookUrl === desiredParams?.webhookUrl && - sameHeaders(actualParams?.headers, desiredParams?.headers) + sameSet(strings(actual.triggerTypes), strings(desired.triggerTypes)) && + sameSet(relationIds(actual.alertSources), relationIds(desired.alertSources)) && + left?.webhookUrl === right?.webhookUrl && + sameSet(headers(left?.headers), headers(right?.headers)) ); } @@ -114,48 +83,42 @@ export async function reconcileIlertAlertAction( ): Promise { const fetchFn = options.fetchFn ?? fetch; const source = alertSource( - await ilertJson( - fetchFn, - options.apiKey, - `/alert-sources/${options.sourceId}`, - ), + await management(fetchFn, options.apiKey, `/alert-sources/${options.sourceId}`), options.sourceId, ); const desired = desiredAlertAction(source, options.webhookSecret); - const listedActions = await listIlertAlertActions( - fetchFn, - options.apiKey, - ); - const actions: unknown[] = []; - for (let index = 0; index < listedActions.length; index += 8) { + const listed = await listActions(fetchFn, options.apiKey); + const actions: Json[] = []; + for (let index = 0; index < listed.length; index += 8) { actions.push( ...(await Promise.all( - listedActions.slice(index, index + 8).map((action) => - ilertJson( - fetchFn, - options.apiKey, - `/alert-actions/${encodeURIComponent(requiredActionId(action))}`, + listed.slice(index, index + 8).map(async (item) => + requireObject( + await management( + fetchFn, + options.apiKey, + `/alert-actions/${encodeURIComponent(actionId(item))}`, + ), + "iLert returned an invalid alert action", ), ), )), ); } - - const candidates = actions.filter((value): value is IlertAlertAction => { - const action = record(value); - if (!action) return false; - const params = record(action.params); + const candidates = actions.filter((action) => { + const params = object(action.params); return action.name === ACTION_NAME || params?.webhookUrl === WEBHOOK_URL; }); - for (const candidate of candidates) { - if ( - candidate.connectorType !== "webhook" || - !sameNumberSet(relationIds(candidate.alertSources), [options.sourceId]) - ) { - throw new Error( - "A conflicting Postil alert action exists; refusing to change its type or source scope", - ); - } + if ( + candidates.some( + (action) => + action.connectorType !== "webhook" || + !sameSet(relationIds(action.alertSources), [options.sourceId]), + ) + ) { + throw new Error( + "A conflicting Postil alert action exists; refusing to change its type or source scope", + ); } if (candidates.length > 1) { throw new Error( @@ -164,31 +127,44 @@ export async function reconcileIlertAlertAction( } const existing = candidates[0]; - if (!existing) { - if (options.dryRun) return { actionId: null, operation: "create" }; - const created = await ilertJson(fetchFn, options.apiKey, "/alert-actions", { - method: "POST", - body: JSON.stringify(desired), - }); - const actionId = requiredActionId(created); - await verifyReconciledAction(fetchFn, options.apiKey, actionId, desired); - return { actionId, operation: "create" }; + const operation: Operation = !existing + ? "create" + : equivalentAlertAction(existing, desired) + ? "unchanged" + : "update"; + if (operation === "unchanged") { + return { actionId: actionId(existing!), operation }; } - - const actionId = requiredActionId(existing); - if (equivalentAlertAction(existing, desired)) { - return { actionId, operation: "unchanged" }; + if (options.dryRun) { + return { actionId: existing ? actionId(existing) : null, operation }; } - if (options.dryRun) return { actionId, operation: "update" }; - const updated = await ilertJson( - fetchFn, - options.apiKey, - `/alert-actions/${encodeURIComponent(actionId)}`, - { method: "PUT", body: JSON.stringify({ ...desired, id: actionId }) }, + + const id = existing ? actionId(existing) : null; + const result = requireObject( + await management( + fetchFn, + options.apiKey, + id ? `/alert-actions/${encodeURIComponent(id)}` : "/alert-actions", + { + method: id ? "PUT" : "POST", + body: JSON.stringify(id ? { ...desired, id } : desired), + }, + ), + "iLert returned an invalid alert action", + ); + const resultId = actionId(result); + const confirmed = requireObject( + await management( + fetchFn, + options.apiKey, + `/alert-actions/${encodeURIComponent(resultId)}`, + ), + "iLert returned an invalid alert action", ); - const updatedId = requiredActionId(updated); - await verifyReconciledAction(fetchFn, options.apiKey, updatedId, desired); - return { actionId: updatedId, operation: "update" }; + if (!equivalentAlertAction(confirmed, desired)) { + throw new Error("iLert did not retain the reconciled alert action"); + } + return { actionId: resultId, operation }; } export async function verifyIlertAlertStreamCanary( @@ -197,89 +173,66 @@ export async function verifyIlertAlertStreamCanary( const fetchFn = options.fetchFn ?? fetch; const sleep = options.sleep ?? Bun.sleep; const startedAt = new Date(Date.now() - 5_000).toISOString(); - const canaryKey = [ + const key = [ "postil-operator-alert-stream-canary", options.runId ?? "local", options.runAttempt ?? "1", randomUUID(), ].join("-"); - let alertAccepted = false; + let accepted = false; let resolveSent = false; - let primaryError: unknown; - + let failure: unknown; try { - await postIlertEvent(fetchFn, { - integrationKey: options.integrationKey, - eventType: "ALERT", - summary: "Postil operator alert stream canary", - details: "GitHub Actions is verifying the Postil operator notification path.", - alertKey: canaryKey, - priority: "HIGH", - }); - alertAccepted = true; - - const created = await waitForCanaryDelivery({ - actionId: options.actionId, - apiKey: options.apiKey, - canaryKey, + await event(fetchFn, options.integrationKey, "ALERT", key); + accepted = true; + const created = await waitForDelivery({ + ...options, fetchFn, + key, sleep, startedAt, }); - await postIlertEvent(fetchFn, { - integrationKey: options.integrationKey, - eventType: "RESOLVE", - summary: "Postil operator alert stream canary resolved", - alertKey: canaryKey, - }); + await event(fetchFn, options.integrationKey, "RESOLVE", key); resolveSent = true; - await waitForCanaryDelivery({ - actionId: options.actionId, + await waitForDelivery({ + ...options, alertId: created.alertId, - apiKey: options.apiKey, - canaryKey, fetchFn, - minimumSuccessfulDeliveries: created.successfulDeliveries + 1, + key, + minimumDeliveries: created.deliveries + 1, requiredStatus: "RESOLVED", sleep, startedAt, }); } catch (error) { - primaryError = error; + failure = error; throw error; } finally { - if (alertAccepted && !resolveSent) { + if (accepted && !resolveSent) { try { - await postIlertEvent(fetchFn, { - integrationKey: options.integrationKey, - eventType: "RESOLVE", - summary: "Postil operator alert stream canary resolved", - alertKey: canaryKey, - }); - } catch (resolveError) { - if (primaryError === undefined) throw resolveError; + await event(fetchFn, options.integrationKey, "RESOLVE", key); + } catch (error) { + if (!failure) throw error; } } } } -interface WaitForCanaryOptions { - actionId: string; +interface WaitOptions extends CanaryOptions { alertId?: string; - apiKey: string; - canaryKey: string; fetchFn: Fetch; - minimumSuccessfulDeliveries?: number; + key: string; + minimumDeliveries?: number; requiredStatus?: string; sleep: Sleep; startedAt: string; } -async function waitForCanaryDelivery( - options: WaitForCanaryOptions, -): Promise<{ alertId: string; successfulDeliveries: number }> { +async function waitForDelivery( + options: WaitOptions, +): Promise<{ alertId: string; deliveries: number }> { for (let attempt = 0; attempt < CANARY_ATTEMPTS; attempt += 1) { - const alerts = await ilertJson( + const alerts = await management( options.fetchFn, options.apiKey, `/alerts?from=${encodeURIComponent(options.startedAt)}&max-results=100`, @@ -287,26 +240,32 @@ async function waitForCanaryDelivery( if (!Array.isArray(alerts)) { throw new Error("iLert returned an invalid alert list during the canary"); } - const alert = alerts.find( - (value) => record(value)?.alertKey === options.canaryKey, + const alert = object( + alerts.find((item) => object(item)?.alertKey === options.key), ); - const alertRecord = record(alert); - const alertId = positiveAlertId(alertRecord?.id); - if (alertId && (!options.alertId || options.alertId === alertId)) { - const actions = await ilertJson( + const id = positiveId(alert?.id); + if (id && (!options.alertId || options.alertId === id)) { + const value = await management( options.fetchFn, options.apiKey, - `/alerts/${encodeURIComponent(alertId)}/actions`, - ); - const successfulDeliveries = successfulActionDeliveries( - actions, - options.actionId, + `/alerts/${encodeURIComponent(id)}/actions`, ); + const actions = Array.isArray(value) ? value : [value]; + if (actions.some((item) => !object(item))) { + throw new Error("iLert returned invalid action history during the canary"); + } + const deliveries = actions + .filter((item) => object(item)?.alertActionId === options.actionId) + .flatMap((item) => { + const history = object(item)?.history; + return Array.isArray(history) ? history : []; + }) + .filter((item) => object(item)?.success === true).length; if ( - successfulDeliveries >= (options.minimumSuccessfulDeliveries ?? 1) && - (!options.requiredStatus || alertRecord?.status === options.requiredStatus) + deliveries >= (options.minimumDeliveries ?? 1) && + (!options.requiredStatus || alert?.status === options.requiredStatus) ) { - return { alertId, successfulDeliveries }; + return { alertId: id, deliveries }; } } if (attempt + 1 < CANARY_ATTEMPTS) await options.sleep(CANARY_RETRY_MS); @@ -314,42 +273,44 @@ async function waitForCanaryDelivery( throw new Error("iLert did not confirm successful Postil webhook delivery"); } -function successfulActionDeliveries(value: unknown, actionId: string): number { - const actions = Array.isArray(value) ? value : [value]; - if (actions.some((action) => !record(action))) { - throw new Error("iLert returned invalid action history during the canary"); - } - return actions - .filter((action) => record(action)?.alertActionId === actionId) - .flatMap((action) => { - const history = record(action)?.history; - if (!Array.isArray(history)) return []; - return history; - }) - .filter((entry) => record(entry)?.success === true).length; -} - -async function postIlertEvent( +async function event( fetchFn: Fetch, - payload: Record, + integrationKey: string, + eventType: "ALERT" | "RESOLVE", + alertKey: string, ): Promise { - const response = await fetchWithTimeout(fetchFn, `${API_BASE}/events`, { + const response = await request(fetchFn, `${API_BASE}/events`, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify(payload), + body: JSON.stringify({ + integrationKey, + eventType, + summary: + eventType === "ALERT" + ? "Postil operator alert stream canary" + : "Postil operator alert stream canary resolved", + ...(eventType === "ALERT" + ? { + details: + "GitHub Actions is verifying the Postil operator notification path.", + priority: "HIGH", + } + : {}), + alertKey, + }), }); if (!response.ok) { throw new Error(`iLert event request failed with HTTP ${response.status}`); } } -async function ilertJson( +async function management( fetchFn: Fetch, apiKey: string, path: string, init: RequestInit = {}, ): Promise { - const response = await fetchWithTimeout(fetchFn, `${API_BASE}${path}`, { + const response = await request(fetchFn, `${API_BASE}${path}`, { ...init, headers: { accept: "application/json", @@ -367,208 +328,161 @@ async function ilertJson( } } -async function listIlertAlertActions( - fetchFn: Fetch, - apiKey: string, -): Promise { - const actions: IlertAlertAction[] = []; - for (let startIndex = 0; startIndex <= 1_000; startIndex += 100) { - const page = await ilertJson( +function request(fetchFn: Fetch, url: string, init: RequestInit): Promise { + return fetchFn(url, { ...init, signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) }); +} + +async function listActions(fetchFn: Fetch, apiKey: string): Promise { + const actions: Json[] = []; + for (let start = 0; start <= 1_000; start += 100) { + const page = await management( fetchFn, apiKey, - `/alert-actions?start-index=${startIndex}&max-results=100`, + `/alert-actions?start-index=${start}&max-results=100`, ); - if (!Array.isArray(page) || page.some((value) => !record(value))) { + if (!Array.isArray(page) || page.some((item) => !object(item))) { throw new Error("iLert returned an invalid alert-action list"); } - actions.push(...(page as IlertAlertAction[])); + actions.push(...(page as Json[])); if (page.length < 100) return actions; } throw new Error("iLert alert-action pagination exceeded the safety bound"); } -async function verifyReconciledAction( - fetchFn: Fetch, - apiKey: string, - actionId: string, - desired: Record, -): Promise { - const confirmed = await ilertJson( - fetchFn, - apiKey, - `/alert-actions/${encodeURIComponent(actionId)}`, - ); - const action = record(confirmed); - if (!action || !equivalentAlertAction(action, desired)) { - throw new Error("iLert did not retain the reconciled alert action"); - } -} - -async function fetchWithTimeout( - fetchFn: Fetch, - url: string, - init: RequestInit, -): Promise { - return fetchFn(url, { - ...init, - signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), - }); -} - -function requiredActionId(value: unknown): string { - const id = opaqueActionId(record(value)?.id); - if (!id) throw new Error("iLert returned an alert action without an identity"); - return id; -} - -function alertSource(value: unknown, expectedId: number): IlertAlertSource { - const source = record(value); - const escalationPolicy = record(source?.escalationPolicy); - const id = positiveSafeNumber(source?.id); - const name = source?.name; - const integrationType = source?.integrationType; - const policyName = escalationPolicy?.name; - const escalationRules = escalationPolicy?.escalationRules; +function alertSource(value: unknown, expectedId: number): Json { + const source = requireObject(value, "iLert returned an invalid alert source"); + const policy = object(source.escalationPolicy); + const id = positiveNumber(source.id); if ( id !== expectedId || - typeof name !== "string" || - name.length === 0 || - typeof integrationType !== "string" || - integrationType.length === 0 || - typeof policyName !== "string" || - policyName.length === 0 || - !Array.isArray(escalationRules) + !nonempty(source.name) || + !nonempty(source.integrationType) || + !nonempty(policy?.name) || + !Array.isArray(policy?.escalationRules) ) { throw new Error("iLert returned an invalid alert source"); } - const policyId = positiveSafeNumber(escalationPolicy?.id); + const policyId = positiveNumber(policy.id); return { id, - name, - integrationType, + name: source.name, + integrationType: source.integrationType, escalationPolicy: { - ...(policyId === null ? {} : { id: policyId }), - name: policyName, - escalationRules, + ...(policyId ? { id: policyId } : {}), + name: policy.name, + escalationRules: policy.escalationRules, }, }; } -function positiveSafeNumber(value: unknown): number | null { - if (typeof value === "number" && Number.isSafeInteger(value) && value > 0) { - return value; - } - if (typeof value === "string" && /^[1-9][0-9]*$/u.test(value)) { - const parsed = Number(value); - if (Number.isSafeInteger(parsed)) return parsed; - } - return null; -} - -function opaqueActionId(value: unknown): string | null { +function actionId(value: unknown): string { + const id = object(value)?.id; + if (typeof id === "number" && Number.isSafeInteger(id) && id > 0) return String(id); if ( - typeof value === "string" && - value.length <= 128 && - /^[A-Za-z0-9._:-]+$/u.test(value) + typeof id === "string" && + id.length <= 128 && + /^[A-Za-z0-9._:-]+$/u.test(id) ) { - return value; - } - if (typeof value === "number" && Number.isSafeInteger(value) && value > 0) { - return String(value); + return id; } - return null; + throw new Error("iLert returned an alert action without an identity"); } -function positiveAlertId(value: unknown): string | null { - if (typeof value === "number" && Number.isSafeInteger(value) && value > 0) { - return String(value); - } - if (typeof value === "string" && /^[1-9][0-9]*$/u.test(value)) return value; - return null; +function positiveId(value: unknown): string | null { + const id = positiveNumber(value); + return id ? String(id) : null; } -function record(value: unknown): Record | null { - return typeof value === "object" && value !== null && !Array.isArray(value) - ? (value as Record) +function positiveNumber(value: unknown): number | null { + const parsed = + typeof value === "string" && /^[1-9][0-9]*$/u.test(value) + ? Number(value) + : value; + return typeof parsed === "number" && Number.isSafeInteger(parsed) && parsed > 0 + ? parsed : null; } function relationIds(value: unknown): number[] { - if (!Array.isArray(value)) return []; - return value.flatMap((entry) => { - const id = record(entry)?.id; - const parsed = positiveSafeNumber(id); - return parsed === null ? [] : [parsed]; - }); + return Array.isArray(value) + ? value.flatMap((item) => { + const id = positiveNumber(object(item)?.id); + return id ? [id] : []; + }) + : []; +} + +function headers(value: unknown): string[] { + return Array.isArray(value) + ? value.flatMap((item) => { + const header = object(item); + return typeof header?.key === "string" && typeof header.value === "string" + ? [`${header.key.toLowerCase()}:${header.value}`] + : []; + }) + : []; } -function sameStringSet(left: unknown, right: unknown): boolean { - if (!Array.isArray(left) || !Array.isArray(right)) return false; - if (!left.every((value) => typeof value === "string")) return false; - if (!right.every((value) => typeof value === "string")) return false; +function strings(value: unknown): string[] { + return Array.isArray(value) && value.every((item) => typeof item === "string") + ? value + : []; +} + +function sameSet(left: T[], right: T[]): boolean { return [...left].sort().join("\0") === [...right].sort().join("\0"); } -function sameNumberSet(left: number[], right: number[]): boolean { - return [...left].sort((a, b) => a - b).join("\0") === - [...right].sort((a, b) => a - b).join("\0"); +function object(value: unknown): Json | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Json) + : null; } -function sameHeaders(left: unknown, right: unknown): boolean { - if (!Array.isArray(left) || !Array.isArray(right) || left.length !== right.length) { - return false; - } - const normalize = (values: unknown[]) => - values - .map((value) => record(value)) - .filter((value): value is Record => value !== null) - .map((value) => `${String(value.key).toLowerCase()}:${String(value.value)}`) - .sort(); - return normalize(left).join("\0") === normalize(right).join("\0"); +function requireObject(value: unknown, message: string): Json { + const result = object(value); + if (!result) throw new Error(message); + return result; } -function requiredEnvironment(name: string): string { - const value = process.env[name]; - if (!value) throw new Error(`${name} is required`); - return value; +function nonempty(value: unknown): value is string { + return typeof value === "string" && value.length > 0; } -function sourceIdFromEnvironment(): number { - const raw = requiredEnvironment("POSTIL_ILERT_ALERT_SOURCE_ID"); - const value = Number(raw); - if (!Number.isSafeInteger(value) || value <= 0 || String(value) !== raw) { - throw new Error("POSTIL_ILERT_ALERT_SOURCE_ID must be a positive integer"); - } +function environment(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); return value; } async function main(): Promise { - const allowed = new Set(["--dry-run", "--canary"]); - if (process.argv.slice(2).some((argument) => !allowed.has(argument))) { + const args = process.argv.slice(2); + if (args.some((arg) => arg !== "--dry-run" && arg !== "--canary")) { throw new Error("usage: reconcile-ilert-alert-stream.ts [--dry-run] [--canary]"); } - const dryRun = process.argv.includes("--dry-run"); - const canary = process.argv.includes("--canary"); + const dryRun = args.includes("--dry-run"); + const canary = args.includes("--canary"); if (dryRun && canary) throw new Error("--canary cannot be combined with --dry-run"); - - const apiKey = requiredEnvironment("ILERT_API_KEY"); + const apiKey = environment("ILERT_API_KEY"); + const sourceId = Number(environment("POSTIL_ILERT_ALERT_SOURCE_ID")); + if (!Number.isSafeInteger(sourceId) || sourceId <= 0) { + throw new Error("POSTIL_ILERT_ALERT_SOURCE_ID must be a positive integer"); + } const result = await reconcileIlertAlertAction({ apiKey, - sourceId: sourceIdFromEnvironment(), - webhookSecret: requiredEnvironment("POSTIL_ILERT_WEBHOOK_SECRET"), + sourceId, + webhookSecret: environment("POSTIL_ILERT_WEBHOOK_SECRET"), dryRun, }); console.log( - dryRun - ? `iLert alert-stream reconciliation plan: ${result.operation}` - : `iLert alert-stream reconciliation: ${result.operation}`, + `iLert alert-stream reconciliation${dryRun ? " plan" : ""}: ${result.operation}`, ); if (canary) { if (!result.actionId) throw new Error("the reconciled alert action has no identity"); await verifyIlertAlertStreamCanary({ actionId: result.actionId, apiKey, - integrationKey: requiredEnvironment("ILERT_INTEGRATION_KEY"), + integrationKey: environment("ILERT_INTEGRATION_KEY"), runId: process.env.GITHUB_RUN_ID, runAttempt: process.env.GITHUB_RUN_ATTEMPT, }); @@ -576,6 +490,4 @@ async function main(): Promise { } } -if (import.meta.main) { - await main(); -} +if (import.meta.main) await main(); From 0d1a69a15bd291b66768c0f52e593edd6b821265 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 16:35:42 +0000 Subject: [PATCH 3/6] Normalize iLert action history identities --- scripts/reconcile-ilert-alert-stream.ts | 25 +++++++++++++++------- tests/reconcile-ilert-alert-stream.test.ts | 2 +- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts/reconcile-ilert-alert-stream.ts b/scripts/reconcile-ilert-alert-stream.ts index 6b522365..252a2b07 100644 --- a/scripts/reconcile-ilert-alert-stream.ts +++ b/scripts/reconcile-ilert-alert-stream.ts @@ -255,7 +255,9 @@ async function waitForDelivery( throw new Error("iLert returned invalid action history during the canary"); } const deliveries = actions - .filter((item) => object(item)?.alertActionId === options.actionId) + .filter( + (item) => opaqueId(object(item)?.alertActionId) === options.actionId, + ) .flatMap((item) => { const history = object(item)?.history; return Array.isArray(history) ? history : []; @@ -376,16 +378,23 @@ function alertSource(value: unknown, expectedId: number): Json { } function actionId(value: unknown): string { - const id = object(value)?.id; - if (typeof id === "number" && Number.isSafeInteger(id) && id > 0) return String(id); + const id = opaqueId(object(value)?.id); + if (id) return id; + throw new Error("iLert returned an alert action without an identity"); +} + +function opaqueId(value: unknown): string | null { + if (typeof value === "number" && Number.isSafeInteger(value) && value > 0) { + return String(value); + } if ( - typeof id === "string" && - id.length <= 128 && - /^[A-Za-z0-9._:-]+$/u.test(id) + typeof value === "string" && + value.length <= 128 && + /^[A-Za-z0-9._:-]+$/u.test(value) ) { - return id; + return value; } - throw new Error("iLert returned an alert action without an identity"); + return null; } function positiveId(value: unknown): string | null { diff --git a/tests/reconcile-ilert-alert-stream.test.ts b/tests/reconcile-ilert-alert-stream.test.ts index 0a356ce4..25a4dfdf 100644 --- a/tests/reconcile-ilert-alert-stream.test.ts +++ b/tests/reconcile-ilert-alert-stream.test.ts @@ -243,7 +243,7 @@ describe("iLert alert-stream reconciliation", () => { (value) => value.method === "POST" && value.url.endsWith("/events"), ).length > 1; return Response.json({ - alertActionId: "72", + alertActionId: 72, history: resolveSent ? [{ success: true }, { success: true }] : [{ success: true }], From 210655cee4b7a7fce1d8a6e30150dc79686e1d65 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 17:29:24 +0000 Subject: [PATCH 4/6] Reconcile terminal review threads From 3d11e5ec9efc1a666fdd29415da2cefcc2a37386 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 14:07:04 +0000 Subject: [PATCH 5/6] Keep canaries out of production alert recovery --- .github/workflows/production-monitor.yml | 2 +- tests/health.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/production-monitor.yml b/.github/workflows/production-monitor.yml index 2317f34e..5516c59d 100644 --- a/.github/workflows/production-monitor.yml +++ b/.github/workflows/production-monitor.yml @@ -667,7 +667,7 @@ jobs: resolve: name: Resolve external alert needs: smoke - if: ${{ needs.smoke.result == 'success' }} + if: ${{ needs.smoke.result == 'success' && inputs.test_alert != true }} permissions: contents: read id-token: write diff --git a/tests/health.test.ts b/tests/health.test.ts index 3bd61300..5a6c46a1 100644 --- a/tests/health.test.ts +++ b/tests/health.test.ts @@ -220,7 +220,7 @@ describe("production monitor workflow", () => { expect(workflow.jobs.resolve?.if).toContain( "needs.smoke.result == 'success'", ); - expect(workflow.jobs.resolve?.if).not.toContain("inputs.test_alert"); + expect(workflow.jobs.resolve?.if).toContain("inputs.test_alert != true"); }); test("enforces monitor health, collection, delivery, failure, and stuck-pass signals", async () => { From 4a654c5315d3508c4c979dd54a2d4ab9e7da249e Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 28 Aug 2026 14:12:45 +0000 Subject: [PATCH 6/6] Clarify iLert canary alert ownership --- .github/workflows/production-monitor.yml | 5 ++++- tests/health.test.ts | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/production-monitor.yml b/.github/workflows/production-monitor.yml index 5516c59d..b35a2059 100644 --- a/.github/workflows/production-monitor.yml +++ b/.github/workflows/production-monitor.yml @@ -583,7 +583,7 @@ jobs: env: POSTIL_ILERT_ALERT_SOURCE_ID: ${{ vars.POSTIL_ILERT_ALERT_SOURCE_ID }} run: bun run scripts/reconcile-ilert-alert-stream.ts --dry-run - - name: Reconcile and verify the iLert alert stream + - name: Reconcile, deliver, and resolve the unique iLert canary env: POSTIL_ILERT_ALERT_SOURCE_ID: ${{ vars.POSTIL_ILERT_ALERT_SOURCE_ID }} run: bun run scripts/reconcile-ilert-alert-stream.ts --canary @@ -664,6 +664,9 @@ jobs: summary: Postil release recovery cleared alert-key: postil-release-recovery + # The alert-stream job resolves its unique canary before it succeeds. This + # resolver owns only the stable production-monitor alert key, so a manual + # canary must not close a production incident. resolve: name: Resolve external alert needs: smoke diff --git a/tests/health.test.ts b/tests/health.test.ts index 5a6c46a1..daa79c5b 100644 --- a/tests/health.test.ts +++ b/tests/health.test.ts @@ -214,7 +214,7 @@ describe("production monitor workflow", () => { "Preview iLert alert-stream reconciliation", ); expect(alertStream?.steps?.map((step) => step.name)).toContain( - "Reconcile and verify the iLert alert stream", + "Reconcile, deliver, and resolve the unique iLert canary", ); expect(workflow.jobs.notify?.if).not.toContain("inputs.test_alert"); expect(workflow.jobs.resolve?.if).toContain(