Reconcile the iLert operator alert stream

.env.example

@@ -27,11 +27,11 @@ POSTIL_SESSION_SECRET=
 GITHUB_WEBHOOK_SECRET=
 
 # Optional iLert outbound alert receiver. Generate a random password with
-# `openssl rand -hex 32`, set it here, and configure the iLert webhook URL as
-# https://postil-ilert:<URL-encoded-password>@<your-host>/api/webhooks/ilert.
-# Select alert-created, alert-assigned, alert-auto-escalated,
-# alert-auto-resolved, alert-acknowledged, alert-rejected, alert-raised, and
-# alert-resolved as trigger events. Leave unset to make the endpoint return 404.
+# `openssl rand -hex 32`, set it here, and configure a Webhook alert action for
+# `https://<your-host>/api/webhooks/ilert`. Send an `Authorization` header with
+# HTTP Basic credentials using `postil-ilert` as the username and this value as
+# the password. Select alert-created, alert-acknowledged, alert-comment-added,
+# and alert-resolved as trigger events. Leave unset to return 404.
 POSTIL_ILERT_WEBHOOK_SECRET=
 
 # Numeric GitHub ids allowed to use operator-only surfaces. The alert stream

.github/workflows/production-monitor.yml

@@ -535,14 +535,67 @@ jobs:
             fi
           fi
 
+  alert-stream:
+    name: Verify operator alert stream
+    needs: smoke
+    if: ${{ inputs.test_alert == true }}
+    permissions:
+      contents: read
+      id-token: write
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+        with:
+          bun-version: 1.3.14
+      - name: Load iLert management secret from Infisical
+        uses: Infisical/secrets-action@77ab1f4ccd183a543cb5b42435fbd181189f4995 # v1.0.16
+        with:
+          method: oidc
+          identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }}
+          project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }}
+          env-slug: prod
+          domain: https://eu.infisical.com
+          secret-path: /
+          secret-name: ILERT_API_KEY
+      - name: Load iLert integration secret from Infisical
+        uses: Infisical/secrets-action@77ab1f4ccd183a543cb5b42435fbd181189f4995 # v1.0.16
+        with:
+          method: oidc
+          identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }}
+          project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }}
+          env-slug: prod
+          domain: https://eu.infisical.com
+          secret-path: /postil
+          secret-name: ILERT_INTEGRATION_KEY
+      - name: Load Postil webhook secret from Infisical
+        uses: Infisical/secrets-action@77ab1f4ccd183a543cb5b42435fbd181189f4995 # v1.0.16
+        with:
+          method: oidc
+          identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }}
+          project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }}
+          env-slug: prod
+          domain: https://eu.infisical.com
+          secret-path: /postil
+          secret-name: POSTIL_ILERT_WEBHOOK_SECRET
+      - name: Preview iLert alert-stream reconciliation
+        env:
+          POSTIL_ILERT_ALERT_SOURCE_ID: ${{ vars.POSTIL_ILERT_ALERT_SOURCE_ID }}
+        run: bun run scripts/reconcile-ilert-alert-stream.ts --dry-run
+      - name: Reconcile, deliver, and resolve the unique iLert canary
+        env:
+          POSTIL_ILERT_ALERT_SOURCE_ID: ${{ vars.POSTIL_ILERT_ALERT_SOURCE_ID }}
+        run: bun run scripts/reconcile-ilert-alert-stream.ts --canary
+
   # Alert delivery is owned by the external alerting service (ilert). The
   # platform and this workflow only report events; paging, escalation, and
   # deduplication happen there. A stable alertKey groups consecutive failing
   # runs into one alert, and the resolve job auto-closes it on recovery.
   notify:
     name: Raise external alert
     needs: [smoke, release-recovery]
-    if: ${{ always() && (needs.smoke.result == 'failure' || needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled' || inputs.test_alert == true) }}
+    if: ${{ always() && (needs.smoke.result == 'failure' || needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') }}
     permissions:
       contents: read
       id-token: write
@@ -563,19 +616,17 @@ jobs:
         uses: ./.github/actions/ilert-event
         with:
           event-type: ALERT
-          summary: ${{ (needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') && 'Postil release recovery failed' || needs.smoke.result == 'failure' && 'Postil production monitor failed' || 'Postil production monitor test alert' }}
-          alert-key: ${{ (needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') && 'postil-release-recovery' || needs.smoke.result == 'failure' && 'postil-production-monitor' || 'postil-production-monitor-test' }}
+          summary: ${{ (needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') && 'Postil release recovery failed' || 'Postil production monitor failed' }}
+          alert-key: ${{ (needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') && 'postil-release-recovery' || 'postil-production-monitor' }}
           details: >-
             ${{ (needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled')
                 && 'Release preparation remains unresolved. Run log:'
-                || needs.smoke.result == 'failure'
-                && 'Production checks failed. Run log:'
-                || 'Operator-requested test alert; production checks passed. Run log:' }}
+                || 'Production checks failed. Run log:' }}
             ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}
           # A routine monitor failure records an alerting gap without masking
-          # the original signal. Recovery failure and test events require
-          # delivery because they validate the fail-safe notification path.
-          require-delivery: ${{ inputs.test_alert == true || needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled' }}
+          # the original signal. Recovery failure requires delivery because it
+          # validates the fail-safe notification path.
+          require-delivery: ${{ needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled' }}
 
   resolve-release-recovery:
     name: Resolve release recovery alert
@@ -613,6 +664,9 @@ jobs:
           summary: Postil release recovery cleared
           alert-key: postil-release-recovery
 
+  # The alert-stream job resolves its unique canary before it succeeds. This
+  # resolver owns only the stable production-monitor alert key, so a manual
+  # canary must not close a production incident.
   resolve:
     name: Resolve external alert
     needs: smoke