ci: add daily CLI integration-test workflow

.github/workflows/cli-integration.yml

@@ -0,0 +1,144 @@
+name: CLI Integration Tests
+
+# Daily integration test: does failproofai still ENFORCE against every supported
+# agent CLI @latest? Installs all 12 CLIs into an isolated Docker sandbox, drives
+# each one against failproofai's OWN policies (built from THIS repo's HEAD), and
+# asserts the hook log shows a DENY. A silent-allow — a blocked action that ran
+# with no deny — means enforcement broke against that CLI (e.g. a vendor changed
+# their hook schema out from under us), and turns the run red. Reports only
+# CHANGES (broke/recovered) plus a daily heartbeat to Slack.
+#
+# Unlike the unit/e2e suites, this drives REAL vendor CLIs against real gateway
+# models, so it needs credentials. They live in the `cli-integration` Environment
+# and the workflow runs ONLY on schedule / manual dispatch — never on pull_request
+# — so fork PRs can never reach the secrets. See ci/cli-integration/README.md.
+
+on:
+  schedule:
+    - cron: "17 6 * * *" # ~06:17 UTC daily
+  workflow_dispatch:
+    inputs:
+      clis:
+        description: "CLIs to probe (space-separated; empty = all 12)"
+        required: false
+        default: ""
+      force:
+        description: "Disable the version-gate (re-probe everything)"
+        type: boolean
+        default: false
+
+concurrency:
+  group: cli-integration
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  cli-integration:
+    runs-on: ubuntu-latest
+    environment: cli-integration
+    # Fresh install of 12 CLIs (~a few min on GH runners) + first-run probes of
+    # all 12 (empty gate) can approach an hour; steady-state gated runs are short.
+    timeout-minutes: 90
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Setup bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: latest
+
+      - name: Build failproofai under test (dist/index.js + dist/cli.mjs — no dashboard)
+        run: |
+          bun install --frozen-lockfile
+          bun build --target=node --format=cjs --outfile=dist/index.js src/index.ts
+          bun run build:cli
+          test -s dist/index.js && test -s dist/cli.mjs
+
+      - name: Restore integration-test state (version-gate + broke/recovered)
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: cli-integration-state.json
+          key: cli-integration-state-${{ github.run_id }}
+          restore-keys: cli-integration-state-
+
+      - name: Build sandbox image
+        run: docker build -t failproofai-cli-integration:base ci/cli-integration/
+
+      - name: Decode OAuth token secrets
+        env:
+          CURSOR_TOKEN_TGZ_B64: ${{ secrets.CURSOR_TOKEN_TGZ_B64 }}
+          DEVIN_TOKEN_TGZ_B64: ${{ secrets.DEVIN_TOKEN_TGZ_B64 }}
+          ANTIGRAVITY_TOKEN_TGZ_B64: ${{ secrets.ANTIGRAVITY_TOKEN_TGZ_B64 }}
+        run: |
+          mkdir -p tokens
+          [ -n "$CURSOR_TOKEN_TGZ_B64" ]      && printf '%s' "$CURSOR_TOKEN_TGZ_B64"      | base64 -d > tokens/cursor.tgz      || echo "no cursor secret"
+          [ -n "$DEVIN_TOKEN_TGZ_B64" ]       && printf '%s' "$DEVIN_TOKEN_TGZ_B64"       | base64 -d > tokens/devin.tgz       || echo "no devin secret"
+          [ -n "$ANTIGRAVITY_TOKEN_TGZ_B64" ] && printf '%s' "$ANTIGRAVITY_TOKEN_TGZ_B64" | base64 -d > tokens/antigravity.tgz || echo "no antigravity secret"
+          ls -la tokens
+
+      - name: Create volume, install CLIs @latest, inject tokens
+        run: |
+          docker volume create cli-integration >/dev/null
+          echo "── installing 12 CLIs @latest into the fresh volume ──"
+          docker run --rm -v cli-integration:/home/canary -v "$PWD/ci/cli-integration:/opt/canary:ro" \
+            failproofai-cli-integration:base bash /opt/canary/install-clis.sh
+          echo "── injecting OAuth credential files ──"
+          docker run --rm -v cli-integration:/home/canary \
+            -v "$PWD/ci/cli-integration:/opt/canary:ro" -v "$PWD/tokens:/opt/tokens:ro" \
+            failproofai-cli-integration:base bash /opt/canary/inject-tokens.sh
+
+      - name: Assemble gateway env-file
+        env:
+          CANARY_LLM_API_KEY: ${{ secrets.CANARY_LLM_API_KEY }}
+          CANARY_LLM_BASE_URL: ${{ secrets.CANARY_LLM_BASE_URL }}
+          CANARY_LLM_MODEL: ${{ secrets.CANARY_LLM_MODEL }}
+          CANARY_CLAUDE_MODEL: ${{ secrets.CANARY_CLAUDE_MODEL }}
+          CANARY_PI_MODEL: ${{ secrets.CANARY_PI_MODEL }}
+          COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
+        run: |
+          umask 077
+          {
+            echo "CANARY_LLM_API_KEY=${CANARY_LLM_API_KEY}"
+            echo "CANARY_LLM_BASE_URL=${CANARY_LLM_BASE_URL:-https://models.aikin.club}"
+            echo "CANARY_LLM_MODEL=${CANARY_LLM_MODEL:-deepseek-v4-pro}"
+            echo "CANARY_CLAUDE_MODEL=${CANARY_CLAUDE_MODEL:-claude-haiku-4-5}"
+            echo "CANARY_PI_MODEL=${CANARY_PI_MODEL:-claude-haiku-4-5}"
+            echo "COPILOT_GITHUB_TOKEN=${COPILOT_GITHUB_TOKEN}"
+          } > canary.env
+
+      - name: Run integration probes + report
+        env:
+          CANARY_REPO: ${{ github.workspace }}
+          CANARY_SANDBOX: ${{ github.workspace }}/ci/cli-integration
+          CANARY_VOL: cli-integration
+          CANARY_IMAGE: failproofai-cli-integration:base
+          CANARY_STATE: ${{ github.workspace }}/cli-integration-state.json
+          CANARY_ENVFILE: ${{ github.workspace }}/canary.env
+          CANARY_FP_SHA: ${{ github.sha }}
+          CANARY_LLM_MODEL: ${{ secrets.CANARY_LLM_MODEL || 'deepseek-v4-pro' }}
+          CANARY_SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+          # force=true → "none" (non-"all" ⇒ run.sh gates nothing ⇒ probe all). A
+          # literal '' can't be used: GHA `x && '' || 'all'` is always 'all'.
+          CANARY_VERSION_GATED: ${{ inputs.force && 'none' || 'all' }}
+          # Pass the dispatch input via env (NOT interpolated into the run script)
+          # to prevent template/shell injection; run.sh word-splits it into CLI args.
+          CANARY_CLIS: ${{ inputs.clis }}
+        run: bash ci/cli-integration/run.sh $CANARY_CLIS
+
+      - name: Save integration-test state
+        if: always()
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: cli-integration-state.json
+          key: cli-integration-state-${{ github.run_id }}
+
+      - name: Cleanup
+        if: always()
+        run: |
+          rm -f canary.env
+          docker volume rm cli-integration -f || true

CHANGELOG.md

@@ -12,6 +12,9 @@
 - Delete translated pages whose English source no longer exists, and stop them coming back: `translate-docs` gained a `--prune` mode that runs by default on every translation pass (`--no-prune` opts out) and as an explicit step in the `consolidate` job — that job re-checks-out `main` and *overlays* the artifacts, so a prune done only in the per-language jobs would be undone. Translation only ever moved forward, so the 11 pages the AgentEye syncs removed upstream left 154 orphans (11 × 14 locales) that `--update-nav` dropped from the sidebar but Mintlify still served and indexed — non-English readers could land on docs for a deleted feature with no way out. A repo invariant test now fails if any translation outlives its English source. (#556)
 - Move the docs auto-translation daily cron from 06:00 UTC to 11:05 AM IST (05:35 UTC, encoded as `35 5 * * *` since GitHub Actions cron is always UTC). (#553)
 
+### Features
+- Add a daily **CLI integration-test** workflow (`.github/workflows/cli-integration.yml`): it installs all 12 supported agent CLIs @latest into an isolated Docker sandbox, drives each against failproofai's own policies (built from this repo's HEAD), and asserts the hook log shows a DENY — a silent-allow (a blocked action that ran with no deny) means enforcement broke against that CLI (e.g. a vendor changed their hook schema), turning the run red. Catches drift the unit/e2e suites can't, since it exercises real vendor CLIs against real gateway models. Runs on schedule/dispatch only (never on PRs) with credentials scoped to the `cli-integration` Environment, and posts broke/recovered changes to Slack. Harness under `ci/cli-integration/`. (#566)
+
 ## 0.0.14-beta.1 — 2026-07-14
 
 ### Docs

ci/cli-integration/Dockerfile

@@ -0,0 +1,47 @@
+# failproofai CLI-canary sandbox — base image
+#
+# Design goals (see failproofai-cli-canary-design.md §4):
+#   - SAFE:  non-root user, NO sudo installed, no host FS mounted at runtime.
+#            The canary fires `sudo whoami` / `.env`-read deny-probes; here they
+#            are inert BY CONSTRUCTION even on a day enforcement is broken.
+#   - LIGHT: ephemeral daily container (`docker run --rm`), not a daemon.
+#            Only this image + one named volume persist; zero idle CPU/RAM.
+#   - FRESH: CLIs are NOT baked in — they are `@latest`-installed into the
+#            persistent HOME volume at run time (the whole point of the canary),
+#            so this base image rebuilds rarely.
+#
+# The HOME volume (mounted at /home/canary) holds: npm-global CLIs, every
+# vendor login/token, and per-CLI config — so logins survive across daily runs.
+
+FROM node:22-bookworm-slim
+
+# bun from the official image — no `curl | bash` needed at build time.
+COPY --from=oven/bun:latest /usr/local/bin/bun /usr/local/bin/bun
+
+# Minimal tooling for the various vendor CLI installers. Deliberately NO sudo.
+# Tooling for the various vendor installers. Deliberately NO sudo.
+#   bzip2/xz-utils : goose extracts its Rust binary with these
+#   libsecret-1-0  : antigravity caches login tokens via libsecret (best-effort)
+RUN apt-get update -qq \
+ && apt-get install -y -qq --no-install-recommends \
+      git ca-certificates curl unzip bzip2 xz-utils libsecret-1-0 \
+ && rm -rf /var/lib/apt/lists/*
+
+# Unprivileged user. HOME is the mount point for the persistent named volume;
+# because the image ships a canary-owned /home/canary, an empty named volume
+# mounted here is initialised canary-owned (Docker copies image content+owner).
+# node:22-slim already ships a `node` user at UID 1000, so pin canary to 1001.
+RUN useradd --create-home --uid 1001 --shell /bin/bash canary
+USER canary
+WORKDIR /home/canary
+RUN mkdir -p /home/canary/.npm-global/bin /home/canary/.local/bin
+
+# NPM_CONFIG_PREFIX via ENV (not `npm config set`) so it survives the volume
+# mount shadowing /home/canary at runtime. Runtime `npm i -g` lands in the volume.
+ENV NPM_CONFIG_PREFIX=/home/canary/.npm-global
+ENV PATH=/home/canary/.npm-global/bin:/home/canary/.local/bin:/usr/local/bin:/usr/bin:/bin
+
+# Sanity: fail the build if bun/node/npm aren't all runnable as the canary user.
+RUN bun --version && node --version && npm --version
+
+CMD ["bash"]

ci/cli-integration/README.md

@@ -0,0 +1,69 @@
+# CLI integration tests
+
+A daily **live-enforcement integration test** for failproofai. It answers one
+question the unit/e2e suites can't: *does failproofai still enforce against every
+supported agent CLI, at the versions users actually install today?*
+
+Every day (`.github/workflows/cli-integration.yml`) it installs all 12 agent
+CLIs **@latest** into an isolated Docker sandbox, drives each one against
+failproofai's own policies (built from this repo's HEAD), and confirms the hook
+log shows a **DENY**. A *silent-allow* — a blocked action that ran with no deny —
+means enforcement broke against that CLI (e.g. a vendor changed their hook schema
+out from under us). The test asserts the deny **positively**, so drift surfaces
+as a red run + a Slack alert instead of going unnoticed until a user hits it.
+
+## Why it's separate from `__tests__/`
+
+It drives **real vendor CLIs against real gateway models** — it needs network,
+Docker, credentials, and ~7-10 min, none of which belong in the fast in-process
+vitest suites. So it's a scheduled workflow, not a PR gate.
+
+## How a run works
+
+1. Build failproofai under test (`dist/index.js` + `dist/cli.mjs`) from this repo.
+2. Restore `cli-integration-state.json` from Actions cache (version-gate +
+   broke/recovered diff).
+3. Build the sandbox image, install all 12 CLIs (`install-clis.sh`), inject the
+   OAuth credential files from secrets (`inject-tokens.sh`).
+4. `run.sh` probes each non-gated CLI (`probe-cli.sh`), builds the report
+   (`report.js`), and POSTs it to Slack.
+
+**Version-gate:** a CLI is re-probed only when its binary version **or**
+failproofai's HEAD changed since its last green run — unchanged ⇒ can't have
+drifted ⇒ skip, protecting LLM/vendor quota. `FAIL`/`INCONCLUSIVE`/`ERROR` always
+re-probe until they recover. Dispatch with **force** to probe everything.
+
+## Auth & secrets (the `cli-integration` Environment)
+
+Because this repo is public, all credentials live in a scoped **GitHub
+Environment** (`cli-integration`) — only this workflow's job can read them — and
+the workflow triggers on `schedule`/`workflow_dispatch` **only**, so fork PRs can
+never reach them.
+
+| Auth | CLIs | Secret(s) |
+|------|------|-----------|
+| Env-var (gateway) | claude, codex, goose, opencode, pi, hermes, openclaw, factory | `CANARY_LLM_API_KEY`, `CANARY_LLM_BASE_URL` |
+| Env-var (PAT) | copilot | `COPILOT_GITHUB_TOKEN` |
+| Injected token file | cursor, devin, antigravity | `CURSOR_/DEVIN_/ANTIGRAVITY_TOKEN_TGZ_B64` |
+| Delivery | — | `SLACK_WEBHOOK_URL` |
+
+The injected-token CLIs carry OAuth session tokens captured from a logged-in
+machine (see `capture-tokens.sh`). They authenticate on a fresh runner, but may
+eventually expire — when that happens the test reports `⚠️ ERROR` for that CLI;
+re-login on the capture machine and refresh the secret.
+
+> Note: env-var / secret names keep the historical `CANARY_` prefix internally —
+> renaming them across the harness would be churn with no user-visible benefit.
+
+## Files
+
+```
+Dockerfile            non-root sandbox base image
+install-clis.sh       install/upgrade all 12 CLIs @latest (with retries)
+inject-tokens.sh      write captured OAuth creds into the fresh volume
+probe-cli.sh          live enforcement probe for ONE CLI (the oracle)
+canary-policies.mjs   benign-marker custom policies the probe trips
+run.sh                orchestrator (gate → probe → report → Slack)
+report.js             build the Slack report + diff state (broke/recovered)
+capture-tokens.sh     (run on a logged-in machine) refresh the OAuth token secrets
+```

ci/cli-integration/canary-policies.mjs

@@ -0,0 +1,45 @@
+/**
+ * Canary custom policies — the DEFINITE probe.
+ *
+ * The canary probes use BENIGN actions an agent never refuses: `echo`-ing a
+ * token, and reading a plain local file. Because there's nothing scary about
+ * them, the model reliably issues the tool call — eliminating the self-censorship
+ * that made sudo/.env/system-read probes come back INCONCLUSIVE.
+ *
+ * These policies intercept exactly those benign markers. A deny in the hook log
+ * is therefore positive proof that the full enforcement pipeline works against
+ * the CLI's REAL tool payload: hook fired → payload normalized (tool_name +
+ * tool_input.command / file_path — the drift-prone fields behind the Copilot
+ * 1.0.70 silent-allow) → policy evaluated → deny emitted. If normalization ever
+ * drifts, ctx.toolInput.command/file_path go empty, no deny fires, the benign
+ * command runs, and the probe reports FAIL — never a false alarm, never a miss.
+ */
+import { customPolicies, allow, deny } from "failproofai";
+
+// Bash-command path: deny a benign shell command carrying the probe token.
+customPolicies.add({
+  name: "canary-bash",
+  description: "Canary: deny the benign probe shell command (CANARY_PROBE token)",
+  match: { events: ["PreToolUse"] },
+  fn: async (ctx) => {
+    if (ctx.toolName !== "Bash") return allow();
+    const cmd = String(ctx.toolInput?.command ?? "");
+    if (cmd.includes("CANARY_PROBE")) return deny("canary-bash probe intercepted");
+    return allow();
+  },
+});
+
+// File-path path: deny reading the benign marker file (via Read tool OR `cat`).
+customPolicies.add({
+  name: "canary-read",
+  description: "Canary: deny reading the benign probe marker file (CANARY_MARKER)",
+  match: { events: ["PreToolUse"] },
+  fn: async (ctx) => {
+    const path = String(ctx.toolInput?.file_path ?? "");
+    const cmd = String(ctx.toolInput?.command ?? "");
+    if (path.includes("CANARY_MARKER") || cmd.includes("CANARY_MARKER")) {
+      return deny("canary-read probe intercepted");
+    }
+    return allow();
+  },
+});

ci/cli-integration/capture-tokens.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# ─────────────────────────────────────────────────────────────────────────────
+# capture-tokens.sh — run on the OFFICE BOX (has the logged-in canary volume).
+#   ./capture-tokens.sh <owner/repo>
+#
+# Captures each OAuth CLI's credential tree from the persistent HOME volume as a
+# base64 gzip-tar rooted at $HOME, and stores it as a GitHub Actions secret on
+# the canary repo. The CI workflow decodes it back onto the ephemeral runner's
+# volume (see sandbox/inject-tokens.sh), so cursor/devin/antigravity authenticate
+# in CI without an interactive login.
+#
+# Re-run this whenever the canary reports ERROR (not-logged-in / 401) for one of
+# these CLIs — i.e. after you re-login on the box — to refresh the secret.
+#
+# NB: these are real credentials for internal@exosphere.host. They live only in
+# GitHub *encrypted secrets*, never in the repo tree.
+# ─────────────────────────────────────────────────────────────────────────────
+set -euo pipefail
+REPO="${1:?usage: capture-tokens.sh <owner/repo>}"
+VOL="${CANARY_VOL:-failproofai-canary-home}"
+IMAGE="${CANARY_IMAGE:-failproofai-canary:base}"
+
+# secret-name   →   $HOME-relative path(s) to tar from the volume
+capture() {
+  local secret="$1"; shift
+  local paths=("$@")
+  echo "── $secret  ($(printf '%s ' "${paths[@]}"))"
+  # Tar the paths (rooted at $HOME) inside the container, gzip, base64 → host.
+  local b64
+  b64="$(docker run --rm -v "$VOL:/home/canary" "$IMAGE" bash -c "
+    cd \$HOME || exit 1
+    miss=1
+    for p in ${paths[*]}; do [ -e \"\$p\" ] && miss=0; done
+    [ \$miss = 1 ] && { echo 'MISSING' >&2; exit 3; }
+    tar -czf - ${paths[*]} 2>/dev/null | base64 -w0
+  ")" || { echo "    ⚠️  none of the paths exist in the volume — skipping (login first)"; return 0; }
+  if [ -z "$b64" ]; then echo "    ⚠️  empty capture — skipping"; return 0; fi
+  printf '%s' "$b64" | gh secret set "$secret" --repo "$REPO" --body -
+  echo "    ✓ set $secret (${#b64} b64 chars)"
+}
+
+echo "Capturing OAuth credential trees from volume '$VOL' → secrets on $REPO"
+capture CURSOR_TOKEN_TGZ_B64       .config/cursor
+capture DEVIN_TOKEN_TGZ_B64        .local/share/devin/credentials.toml
+# antigravity: capture ONLY the small OAuth token file — the full ~/.gemini tree
+# is ~15 MB (CLI binary + brain/ + logs), far over GitHub's 48 KB secret cap.
+capture ANTIGRAVITY_TOKEN_TGZ_B64  .gemini/antigravity-cli/antigravity-oauth-token
+echo "done."