From dbcc1a8fcbba748c6e26e5a5722dcf454416799d Mon Sep 17 00:00:00 2001 From: "Nivedit (Hermes)" Date: Fri, 17 Jul 2026 16:02:13 +0530 Subject: [PATCH 1/3] ci: add daily CLI integration-test workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Installs all 12 supported agent CLIs @latest into an isolated Docker sandbox, drives each against failproofai's own policies (built from this repo's HEAD), and asserts the hook log shows a DENY — a silent-allow means enforcement broke against that CLI (e.g. vendor hook-schema drift). Ports the standalone canary into the repo as a first-class integration test. - .github/workflows/cli-integration.yml: schedule + workflow_dispatch only (never PRs), credentials scoped to the cli-integration Environment (public repo), SHA-pinned actions. - ci/cli-integration/: non-root Docker sandbox + probe/install/inject/report harness (token-injection lets cursor/devin/antigravity auth on the runner). - eslint: ignore ci/ (standalone Node+shell harness, not shipped source). - CHANGELOG entry. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/cli-integration.yml | 141 +++++++++++++++++ CHANGELOG.md | 1 + ci/cli-integration/Dockerfile | 47 ++++++ ci/cli-integration/README.md | 69 +++++++++ ci/cli-integration/canary-policies.mjs | 45 ++++++ ci/cli-integration/capture-tokens.sh | 48 ++++++ ci/cli-integration/inject-tokens.sh | 44 ++++++ ci/cli-integration/install-clis.sh | 91 +++++++++++ ci/cli-integration/probe-cli.sh | 202 +++++++++++++++++++++++++ ci/cli-integration/report.js | 69 +++++++++ ci/cli-integration/run.sh | 94 ++++++++++++ eslint.config.mjs | 4 +- 12 files changed, 854 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/cli-integration.yml create mode 100644 ci/cli-integration/Dockerfile create mode 100644 ci/cli-integration/README.md create mode 100644 ci/cli-integration/canary-policies.mjs create mode 100644 ci/cli-integration/capture-tokens.sh create mode 100644 ci/cli-integration/inject-tokens.sh create mode 100644 ci/cli-integration/install-clis.sh create mode 100644 ci/cli-integration/probe-cli.sh create mode 100644 ci/cli-integration/report.js create mode 100644 ci/cli-integration/run.sh diff --git a/.github/workflows/cli-integration.yml b/.github/workflows/cli-integration.yml new file mode 100644 index 00000000..aa805994 --- /dev/null +++ b/.github/workflows/cli-integration.yml @@ -0,0 +1,141 @@ +name: CLI Integration Tests + +# Daily integration test: does failproofai still ENFORCE against every supported +# agent CLI @latest? Installs all 12 CLIs into an isolated Docker sandbox, drives +# each one against failproofai's OWN policies (built from THIS repo's HEAD), and +# asserts the hook log shows a DENY. A silent-allow — a blocked action that ran +# with no deny — means enforcement broke against that CLI (e.g. a vendor changed +# their hook schema out from under us), and turns the run red. Reports only +# CHANGES (broke/recovered) plus a daily heartbeat to Slack. +# +# Unlike the unit/e2e suites, this drives REAL vendor CLIs against real gateway +# models, so it needs credentials. They live in the `cli-integration` Environment +# and the workflow runs ONLY on schedule / manual dispatch — never on pull_request +# — so fork PRs can never reach the secrets. See ci/cli-integration/README.md. + +on: + schedule: + - cron: "17 6 * * *" # ~06:17 UTC daily + workflow_dispatch: + inputs: + clis: + description: "CLIs to probe (space-separated; empty = all 12)" + required: false + default: "" + force: + description: "Disable the version-gate (re-probe everything)" + type: boolean + default: false + +concurrency: + group: cli-integration + cancel-in-progress: false + +permissions: + contents: read + +jobs: + cli-integration: + runs-on: ubuntu-latest + environment: cli-integration + # Fresh install of 12 CLIs (~a few min on GH runners) + first-run probes of + # all 12 (empty gate) can approach an hour; steady-state gated runs are short. + timeout-minutes: 90 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Setup bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: latest + + - name: Build failproofai under test (dist/index.js + dist/cli.mjs — no dashboard) + run: | + bun install --frozen-lockfile + bun build --target=node --format=cjs --outfile=dist/index.js src/index.ts + bun run build:cli + test -s dist/index.js && test -s dist/cli.mjs + + - name: Restore integration-test state (version-gate + broke/recovered) + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: cli-integration-state.json + key: cli-integration-state-${{ github.run_id }} + restore-keys: cli-integration-state- + + - name: Build sandbox image + run: docker build -t failproofai-cli-integration:base ci/cli-integration/ + + - name: Decode OAuth token secrets + env: + CURSOR_TOKEN_TGZ_B64: ${{ secrets.CURSOR_TOKEN_TGZ_B64 }} + DEVIN_TOKEN_TGZ_B64: ${{ secrets.DEVIN_TOKEN_TGZ_B64 }} + ANTIGRAVITY_TOKEN_TGZ_B64: ${{ secrets.ANTIGRAVITY_TOKEN_TGZ_B64 }} + run: | + mkdir -p tokens + [ -n "$CURSOR_TOKEN_TGZ_B64" ] && printf '%s' "$CURSOR_TOKEN_TGZ_B64" | base64 -d > tokens/cursor.tgz || echo "no cursor secret" + [ -n "$DEVIN_TOKEN_TGZ_B64" ] && printf '%s' "$DEVIN_TOKEN_TGZ_B64" | base64 -d > tokens/devin.tgz || echo "no devin secret" + [ -n "$ANTIGRAVITY_TOKEN_TGZ_B64" ] && printf '%s' "$ANTIGRAVITY_TOKEN_TGZ_B64" | base64 -d > tokens/antigravity.tgz || echo "no antigravity secret" + ls -la tokens + + - name: Create volume, install CLIs @latest, inject tokens + run: | + docker volume create cli-integration >/dev/null + echo "── installing 12 CLIs @latest into the fresh volume ──" + docker run --rm -v cli-integration:/home/canary -v "$PWD/ci/cli-integration:/opt/canary:ro" \ + failproofai-cli-integration:base bash /opt/canary/install-clis.sh + echo "── injecting OAuth credential files ──" + docker run --rm -v cli-integration:/home/canary \ + -v "$PWD/ci/cli-integration:/opt/canary:ro" -v "$PWD/tokens:/opt/tokens:ro" \ + failproofai-cli-integration:base bash /opt/canary/inject-tokens.sh + + - name: Assemble gateway env-file + env: + CANARY_LLM_API_KEY: ${{ secrets.CANARY_LLM_API_KEY }} + CANARY_LLM_BASE_URL: ${{ secrets.CANARY_LLM_BASE_URL }} + CANARY_LLM_MODEL: ${{ secrets.CANARY_LLM_MODEL }} + CANARY_CLAUDE_MODEL: ${{ secrets.CANARY_CLAUDE_MODEL }} + CANARY_PI_MODEL: ${{ secrets.CANARY_PI_MODEL }} + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + run: | + umask 077 + { + echo "CANARY_LLM_API_KEY=${CANARY_LLM_API_KEY}" + echo "CANARY_LLM_BASE_URL=${CANARY_LLM_BASE_URL:-https://models.aikin.club}" + echo "CANARY_LLM_MODEL=${CANARY_LLM_MODEL:-deepseek-v4-pro}" + echo "CANARY_CLAUDE_MODEL=${CANARY_CLAUDE_MODEL:-claude-haiku-4-5}" + echo "CANARY_PI_MODEL=${CANARY_PI_MODEL:-claude-haiku-4-5}" + echo "COPILOT_GITHUB_TOKEN=${COPILOT_GITHUB_TOKEN}" + } > canary.env + + - name: Run integration probes + report + env: + CANARY_REPO: ${{ github.workspace }} + CANARY_SANDBOX: ${{ github.workspace }}/ci/cli-integration + CANARY_VOL: cli-integration + CANARY_IMAGE: failproofai-cli-integration:base + CANARY_STATE: ${{ github.workspace }}/cli-integration-state.json + CANARY_ENVFILE: ${{ github.workspace }}/canary.env + CANARY_FP_SHA: ${{ github.sha }} + CANARY_LLM_MODEL: ${{ secrets.CANARY_LLM_MODEL || 'deepseek-v4-pro' }} + CANARY_SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + # force=true → "none" (non-"all" ⇒ run.sh gates nothing ⇒ probe all). A + # literal '' can't be used: GHA `x && '' || 'all'` is always 'all'. + CANARY_VERSION_GATED: ${{ inputs.force && 'none' || 'all' }} + run: bash ci/cli-integration/run.sh ${{ inputs.clis }} + + - name: Save integration-test state + if: always() + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: cli-integration-state.json + key: cli-integration-state-${{ github.run_id }} + + - name: Cleanup + if: always() + run: | + rm -f canary.env + docker volume rm cli-integration -f || true diff --git a/CHANGELOG.md b/CHANGELOG.md index a2450f93..b7b9c8f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 0.0.14-beta.1 — 2026-07-14 ### Features +- Add a daily **CLI integration-test** workflow (`.github/workflows/cli-integration.yml`): it installs all 12 supported agent CLIs @latest into an isolated Docker sandbox, drives each against failproofai's own policies (built from this repo's HEAD), and asserts the hook log shows a DENY — a silent-allow (a blocked action that ran with no deny) means enforcement broke against that CLI (e.g. a vendor changed their hook schema), turning the run red. Catches drift the unit/e2e suites can't, since it exercises real vendor CLIs against real gateway models. Runs on schedule/dispatch only (never on PRs) with credentials scoped to the `cli-integration` Environment, and posts broke/recovered changes to Slack. Harness under `ci/cli-integration/`. (#PRNUM) - Brand the dashboard launch splash (bare `failproofai`) to match the `configure` wizard: the plain emoji title/version block is replaced with the half-block logomark, the `failproof ai` wordmark (pink "il"), the tagline, and a tidy teal-labelled version/links column (24-bit color where advertised, monochrome shape otherwise, plain text off a TTY). The logomark rendering is now shared via `renderBrandLogo` / `renderLaunchBanner` in `tui.ts`, so the wizard intro and the launch banner stay in lockstep. (#516) - Add an "Everything available" row to the wizard's assistants step that protects every supported CLI (detected + set-up-ahead) in one tick — it wins over the individual boxes, mirroring the policies "Everything" option. Detected CLIs stay pre-selected, so the default (hit ↵) is unchanged. (#516) - Make the wizard's "What should we guard against?" step a multi-select, so bundles combine: tick any mix of Secrets & data / Git safety / Ship discipline / Cloud & infra and the enabled set is the **union** of their policies (deduped), or tick **Everything** for the full set (it wins over any presets). Dropped only the "Custom…" entry — an action-as-checkbox that didn't fit the list (the full searchable picker stays available via `failproofai policies --install`). Replaces the single-choice radio; `resolvePolicySource` → `resolvePresetSelection`. (#516) diff --git a/ci/cli-integration/Dockerfile b/ci/cli-integration/Dockerfile new file mode 100644 index 00000000..e4aa6cd7 --- /dev/null +++ b/ci/cli-integration/Dockerfile @@ -0,0 +1,47 @@ +# failproofai CLI-canary sandbox — base image +# +# Design goals (see failproofai-cli-canary-design.md §4): +# - SAFE: non-root user, NO sudo installed, no host FS mounted at runtime. +# The canary fires `sudo whoami` / `.env`-read deny-probes; here they +# are inert BY CONSTRUCTION even on a day enforcement is broken. +# - LIGHT: ephemeral daily container (`docker run --rm`), not a daemon. +# Only this image + one named volume persist; zero idle CPU/RAM. +# - FRESH: CLIs are NOT baked in — they are `@latest`-installed into the +# persistent HOME volume at run time (the whole point of the canary), +# so this base image rebuilds rarely. +# +# The HOME volume (mounted at /home/canary) holds: npm-global CLIs, every +# vendor login/token, and per-CLI config — so logins survive across daily runs. + +FROM node:22-bookworm-slim + +# bun from the official image — no `curl | bash` needed at build time. +COPY --from=oven/bun:latest /usr/local/bin/bun /usr/local/bin/bun + +# Minimal tooling for the various vendor CLI installers. Deliberately NO sudo. +# Tooling for the various vendor installers. Deliberately NO sudo. +# bzip2/xz-utils : goose extracts its Rust binary with these +# libsecret-1-0 : antigravity caches login tokens via libsecret (best-effort) +RUN apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends \ + git ca-certificates curl unzip bzip2 xz-utils libsecret-1-0 \ + && rm -rf /var/lib/apt/lists/* + +# Unprivileged user. HOME is the mount point for the persistent named volume; +# because the image ships a canary-owned /home/canary, an empty named volume +# mounted here is initialised canary-owned (Docker copies image content+owner). +# node:22-slim already ships a `node` user at UID 1000, so pin canary to 1001. +RUN useradd --create-home --uid 1001 --shell /bin/bash canary +USER canary +WORKDIR /home/canary +RUN mkdir -p /home/canary/.npm-global/bin /home/canary/.local/bin + +# NPM_CONFIG_PREFIX via ENV (not `npm config set`) so it survives the volume +# mount shadowing /home/canary at runtime. Runtime `npm i -g` lands in the volume. +ENV NPM_CONFIG_PREFIX=/home/canary/.npm-global +ENV PATH=/home/canary/.npm-global/bin:/home/canary/.local/bin:/usr/local/bin:/usr/bin:/bin + +# Sanity: fail the build if bun/node/npm aren't all runnable as the canary user. +RUN bun --version && node --version && npm --version + +CMD ["bash"] diff --git a/ci/cli-integration/README.md b/ci/cli-integration/README.md new file mode 100644 index 00000000..1fe5d1d2 --- /dev/null +++ b/ci/cli-integration/README.md @@ -0,0 +1,69 @@ +# CLI integration tests + +A daily **live-enforcement integration test** for failproofai. It answers one +question the unit/e2e suites can't: *does failproofai still enforce against every +supported agent CLI, at the versions users actually install today?* + +Every day (`.github/workflows/cli-integration.yml`) it installs all 12 agent +CLIs **@latest** into an isolated Docker sandbox, drives each one against +failproofai's own policies (built from this repo's HEAD), and confirms the hook +log shows a **DENY**. A *silent-allow* — a blocked action that ran with no deny — +means enforcement broke against that CLI (e.g. a vendor changed their hook schema +out from under us). The test asserts the deny **positively**, so drift surfaces +as a red run + a Slack alert instead of going unnoticed until a user hits it. + +## Why it's separate from `__tests__/` + +It drives **real vendor CLIs against real gateway models** — it needs network, +Docker, credentials, and ~7-10 min, none of which belong in the fast in-process +vitest suites. So it's a scheduled workflow, not a PR gate. + +## How a run works + +1. Build failproofai under test (`dist/index.js` + `dist/cli.mjs`) from this repo. +2. Restore `cli-integration-state.json` from Actions cache (version-gate + + broke/recovered diff). +3. Build the sandbox image, install all 12 CLIs (`install-clis.sh`), inject the + OAuth credential files from secrets (`inject-tokens.sh`). +4. `run.sh` probes each non-gated CLI (`probe-cli.sh`), builds the report + (`report.js`), and POSTs it to Slack. + +**Version-gate:** a CLI is re-probed only when its binary version **or** +failproofai's HEAD changed since its last green run — unchanged ⇒ can't have +drifted ⇒ skip, protecting LLM/vendor quota. `FAIL`/`INCONCLUSIVE`/`ERROR` always +re-probe until they recover. Dispatch with **force** to probe everything. + +## Auth & secrets (the `cli-integration` Environment) + +Because this repo is public, all credentials live in a scoped **GitHub +Environment** (`cli-integration`) — only this workflow's job can read them — and +the workflow triggers on `schedule`/`workflow_dispatch` **only**, so fork PRs can +never reach them. + +| Auth | CLIs | Secret(s) | +|------|------|-----------| +| Env-var (gateway) | claude, codex, goose, opencode, pi, hermes, openclaw, factory | `CANARY_LLM_API_KEY`, `CANARY_LLM_BASE_URL` | +| Env-var (PAT) | copilot | `COPILOT_GITHUB_TOKEN` | +| Injected token file | cursor, devin, antigravity | `CURSOR_/DEVIN_/ANTIGRAVITY_TOKEN_TGZ_B64` | +| Delivery | — | `SLACK_WEBHOOK_URL` | + +The injected-token CLIs carry OAuth session tokens captured from a logged-in +machine (see `capture-tokens.sh`). They authenticate on a fresh runner, but may +eventually expire — when that happens the test reports `⚠️ ERROR` for that CLI; +re-login on the capture machine and refresh the secret. + +> Note: env-var / secret names keep the historical `CANARY_` prefix internally — +> renaming them across the harness would be churn with no user-visible benefit. + +## Files + +``` +Dockerfile non-root sandbox base image +install-clis.sh install/upgrade all 12 CLIs @latest (with retries) +inject-tokens.sh write captured OAuth creds into the fresh volume +probe-cli.sh live enforcement probe for ONE CLI (the oracle) +canary-policies.mjs benign-marker custom policies the probe trips +run.sh orchestrator (gate → probe → report → Slack) +report.js build the Slack report + diff state (broke/recovered) +capture-tokens.sh (run on a logged-in machine) refresh the OAuth token secrets +``` diff --git a/ci/cli-integration/canary-policies.mjs b/ci/cli-integration/canary-policies.mjs new file mode 100644 index 00000000..0b1004d2 --- /dev/null +++ b/ci/cli-integration/canary-policies.mjs @@ -0,0 +1,45 @@ +/** + * Canary custom policies — the DEFINITE probe. + * + * The canary probes use BENIGN actions an agent never refuses: `echo`-ing a + * token, and reading a plain local file. Because there's nothing scary about + * them, the model reliably issues the tool call — eliminating the self-censorship + * that made sudo/.env/system-read probes come back INCONCLUSIVE. + * + * These policies intercept exactly those benign markers. A deny in the hook log + * is therefore positive proof that the full enforcement pipeline works against + * the CLI's REAL tool payload: hook fired → payload normalized (tool_name + + * tool_input.command / file_path — the drift-prone fields behind the Copilot + * 1.0.70 silent-allow) → policy evaluated → deny emitted. If normalization ever + * drifts, ctx.toolInput.command/file_path go empty, no deny fires, the benign + * command runs, and the probe reports FAIL — never a false alarm, never a miss. + */ +import { customPolicies, allow, deny } from "failproofai"; + +// Bash-command path: deny a benign shell command carrying the probe token. +customPolicies.add({ + name: "canary-bash", + description: "Canary: deny the benign probe shell command (CANARY_PROBE token)", + match: { events: ["PreToolUse"] }, + fn: async (ctx) => { + if (ctx.toolName !== "Bash") return allow(); + const cmd = String(ctx.toolInput?.command ?? ""); + if (cmd.includes("CANARY_PROBE")) return deny("canary-bash probe intercepted"); + return allow(); + }, +}); + +// File-path path: deny reading the benign marker file (via Read tool OR `cat`). +customPolicies.add({ + name: "canary-read", + description: "Canary: deny reading the benign probe marker file (CANARY_MARKER)", + match: { events: ["PreToolUse"] }, + fn: async (ctx) => { + const path = String(ctx.toolInput?.file_path ?? ""); + const cmd = String(ctx.toolInput?.command ?? ""); + if (path.includes("CANARY_MARKER") || cmd.includes("CANARY_MARKER")) { + return deny("canary-read probe intercepted"); + } + return allow(); + }, +}); diff --git a/ci/cli-integration/capture-tokens.sh b/ci/cli-integration/capture-tokens.sh new file mode 100644 index 00000000..55424043 --- /dev/null +++ b/ci/cli-integration/capture-tokens.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# capture-tokens.sh — run on the OFFICE BOX (has the logged-in canary volume). +# ./capture-tokens.sh +# +# Captures each OAuth CLI's credential tree from the persistent HOME volume as a +# base64 gzip-tar rooted at $HOME, and stores it as a GitHub Actions secret on +# the canary repo. The CI workflow decodes it back onto the ephemeral runner's +# volume (see sandbox/inject-tokens.sh), so cursor/devin/antigravity authenticate +# in CI without an interactive login. +# +# Re-run this whenever the canary reports ERROR (not-logged-in / 401) for one of +# these CLIs — i.e. after you re-login on the box — to refresh the secret. +# +# NB: these are real credentials for internal@exosphere.host. They live only in +# GitHub *encrypted secrets*, never in the repo tree. +# ───────────────────────────────────────────────────────────────────────────── +set -euo pipefail +REPO="${1:?usage: capture-tokens.sh }" +VOL="${CANARY_VOL:-failproofai-canary-home}" +IMAGE="${CANARY_IMAGE:-failproofai-canary:base}" + +# secret-name → $HOME-relative path(s) to tar from the volume +capture() { + local secret="$1"; shift + local paths=("$@") + echo "── $secret ($(printf '%s ' "${paths[@]}"))" + # Tar the paths (rooted at $HOME) inside the container, gzip, base64 → host. + local b64 + b64="$(docker run --rm -v "$VOL:/home/canary" "$IMAGE" bash -c " + cd \$HOME || exit 1 + miss=1 + for p in ${paths[*]}; do [ -e \"\$p\" ] && miss=0; done + [ \$miss = 1 ] && { echo 'MISSING' >&2; exit 3; } + tar -czf - ${paths[*]} 2>/dev/null | base64 -w0 + ")" || { echo " ⚠️ none of the paths exist in the volume — skipping (login first)"; return 0; } + if [ -z "$b64" ]; then echo " ⚠️ empty capture — skipping"; return 0; fi + printf '%s' "$b64" | gh secret set "$secret" --repo "$REPO" --body - + echo " ✓ set $secret (${#b64} b64 chars)" +} + +echo "Capturing OAuth credential trees from volume '$VOL' → secrets on $REPO" +capture CURSOR_TOKEN_TGZ_B64 .config/cursor +capture DEVIN_TOKEN_TGZ_B64 .local/share/devin/credentials.toml +# antigravity: capture ONLY the small OAuth token file — the full ~/.gemini tree +# is ~15 MB (CLI binary + brain/ + logs), far over GitHub's 48 KB secret cap. +capture ANTIGRAVITY_TOKEN_TGZ_B64 .gemini/antigravity-cli/antigravity-oauth-token +echo "done." diff --git a/ci/cli-integration/inject-tokens.sh b/ci/cli-integration/inject-tokens.sh new file mode 100644 index 00000000..f6c581df --- /dev/null +++ b/ci/cli-integration/inject-tokens.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# Token injection — runs INSIDE the sandbox container, against the fresh per-run +# HOME volume, BEFORE the probes. +# +# On the office box the OAuth logins (cursor / devin / antigravity) live in a +# PERSISTENT volume. GitHub-Actions runners are ephemeral, so instead each of +# those credential trees is captured once (see capture-tokens.sh) into a GitHub +# secret as a base64 gzip-tar rooted at $HOME, decoded by the workflow into +# /opt/tokens/.tgz, and extracted here into the fresh volume's $HOME. +# +# Extraction is rooted at $HOME and the tarballs were created with $HOME-relative +# paths (e.g. `.config/cursor`, `.local/share/devin/credentials.toml`), so each +# file lands exactly where its CLI expects it. Env-var-auth CLIs (gateway key, +# copilot PAT, factory BYOK) need NO file here — they authenticate from env. +# ───────────────────────────────────────────────────────────────────────────── +set -u +TOK="${1:-/opt/tokens}" + +if [ ! -d "$TOK" ]; then + echo "inject-tokens: no token dir at $TOK — nothing to inject (env-var CLIs only)" + exit 0 +fi + +shopt -s nullglob +found=0 +for t in "$TOK"/*.tgz "$TOK"/*.tar.gz; do + found=1 + echo "── injecting $(basename "$t") → \$HOME ──" + if tar -C "$HOME" -xzf "$t"; then + tar -tzf "$t" 2>/dev/null | sed 's/^/ /' | head -8 + else + echo " WARN: extract failed for $t (skipping — that CLI will report ERROR)" + fi +done +[ "$found" = 0 ] && echo "inject-tokens: token dir present but empty — env-var CLIs only" + +echo "── credential files now present in volume ──" +for f in "$HOME/.config/cursor/auth.json" \ + "$HOME/.local/share/devin/credentials.toml" \ + "$HOME/.gemini/antigravity-cli"; do + [ -e "$f" ] && echo " ✓ $f" || echo " ✗ $f (absent)" +done +exit 0 diff --git a/ci/cli-integration/install-clis.sh b/ci/cli-integration/install-clis.sh new file mode 100644 index 00000000..1d3f8213 --- /dev/null +++ b/ci/cli-integration/install-clis.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# Tier-0 install probe — runs INSIDE the canary sandbox container. +# +# Installs/upgrades all 12 agent CLIs to @latest into the persistent HOME volume, +# then records which resolve to a runnable binary + their version. This is both +# the daily "fresh upgrade" step AND the Tier-0 canary signal (packaging/binary +# breaks show up here as a FAIL). +# +# Install methods verified 2026-07-16 (see failproofai-cli-canary-design.md). +# Vendor installers scatter binaries across several dirs, so we search an +# expanded PATH and fall back to a bounded `find` under $HOME. +# +# Output: a JSON array at $CANARY_RESULTS (default ~/canary-tier0.json) + a +# human table on stdout. Never `set -e` — one CLI failing must not abort the rest. +# ───────────────────────────────────────────────────────────────────────────── +set -u + +RESULTS="${CANARY_RESULTS:-$HOME/canary-tier0.json}" +LOGDIR="${CANARY_LOGDIR:-$HOME/canary-logs}" +mkdir -p "$LOGDIR" + +# Cover every dir the vendor installers are known to drop binaries into. +export PATH="$HOME/.npm-global/bin:$HOME/.local/bin:$HOME/.factory/bin:$HOME/.hermes/bin:$HOME/.cursor/bin:$HOME/.codex/bin:$HOME/bin:$PATH" + +ROWS=() + +probe() { + local id="$1" bin="$2" vflag="$3" install="$4" + echo "════════════════════ $id ════════════════════" + local t0 t1 rc=1; t0=$SECONDS + # Retry transient install failures. Flaky vendor CDNs / HTTP-2 stream resets are + # common on ephemeral CI runners (e.g. factory's `curl … | sh` dropping the + # download mid-stream with `curl: (18)`). Break as soon as the binary resolves; + # otherwise give it up to 3 attempts so one bad download isn't a false FAIL. + for attempt in 1 2 3; do + bash -c "$install" >"$LOGDIR/$id.install.log" 2>&1 + rc=$? + hash -r 2>/dev/null + command -v "$bin" >/dev/null 2>&1 && break + find "$HOME" -maxdepth 5 -name "$bin" -type f -perm -u+x 2>/dev/null | grep -q . && break + [ "$attempt" -lt 3 ] && { echo " ↻ $id install attempt $attempt failed (rc=$rc) — retrying in 5s"; sleep 5; } + done + t1=$SECONDS + hash -r 2>/dev/null + + # Resolve the binary: PATH first, then a bounded find under $HOME. + local resolved; resolved="$(command -v "$bin" 2>/dev/null || true)" + if [ -z "$resolved" ]; then + resolved="$(find "$HOME" -maxdepth 5 -name "$bin" -type f -perm -u+x 2>/dev/null | head -1)" + fi + + local ver="" status="FAIL" + if [ -n "$resolved" ]; then + status="OK" + ver="$("$resolved" $vflag 2>&1 | head -1 | tr -cd '[:alnum:] ._+:/-')" + fi + + printf ' rc=%s time=%ss bin=%s ver=%s => %s\n' \ + "$rc" "$((t1 - t0))" "${resolved:-}" "${ver:-}" "$status" + [ "$status" = FAIL ] && echo " ↳ last install log lines:" && tail -3 "$LOGDIR/$id.install.log" | sed 's/^/ /' + + ROWS+=("$(printf '{"cli":"%s","status":"%s","binary":"%s","version":"%s","install_rc":%s,"secs":%s}' \ + "$id" "$status" "${resolved:-}" "${ver:-}" "$rc" "$((t1 - t0))")") +} + +# id binary --ver flag install command (@latest) +probe claude claude "--version" 'curl -fsSL https://claude.ai/install.sh | bash' +probe codex codex "--version" 'npm install -g @openai/codex@latest' +probe copilot copilot "--version" 'npm install -g @github/copilot@latest' +probe cursor cursor-agent "--version" 'curl https://cursor.com/install -fsS | bash' +probe opencode opencode "--version" 'npm install -g opencode-ai@latest' +probe pi pi "--version" 'npm install -g @mariozechner/pi-coding-agent@latest' +probe hermes hermes "--version" 'curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash' +probe openclaw openclaw "--version" 'npm install -g openclaw@latest' +probe factory droid "--version" 'curl -fsSL https://app.factory.ai/cli | sh' +probe devin devin "--version" 'curl -fsSL https://cli.devin.ai/install.sh | bash' +probe antigravity agy "--version" 'curl -fsSL https://antigravity.google/cli/install.sh | bash' +probe goose goose "--version" 'curl -fsSL https://github.com/aaif-goose/goose/releases/download/stable/download_cli.sh | CONFIGURE=false bash' + +# Emit JSON array. +{ printf '['; IFS=,; printf '%s' "${ROWS[*]}"; printf ']\n'; } > "$RESULTS" + +echo +echo "════════════════════ TIER-0 SUMMARY ════════════════════" +ok=0; fail=0 +for r in "${ROWS[@]}"; do + case "$r" in *'"status":"OK"'*) ok=$((ok+1));; *) fail=$((fail+1));; esac +done +printf 'installed OK: %s / 12 failed: %s\n' "$ok" "$fail" +echo "results JSON: $RESULTS" diff --git a/ci/cli-integration/probe-cli.sh b/ci/cli-integration/probe-cli.sh new file mode 100644 index 00000000..54a19ce2 --- /dev/null +++ b/ci/cli-integration/probe-cli.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# Live Tier-2 enforcement probe for ONE CLI. Runs INSIDE the sandbox container. +# Usage: probe-cli.sh +# +# Hooks via failproofai's OWN installer, pointed at repo main HEAD (a wrapper + +# FAILPROOFAI_BINARY_OVERRIDE at user scope; project-scope fallback swaps the +# `npx -y failproofai` command to `bun /repo/…`). Only wire()/drive() are per-CLI. +# +# Verdict (design 3-way): PASS = oracle log shows expected deny; FAIL = forbidden +# side-effect leaked; INCONCLUSIVE = model never attempted the tool. LOG_LEVEL=info +# is mandatory (oracle lines are INFO). +# ───────────────────────────────────────────────────────────────────────────── +set -u +CLI="${1:?usage: probe-cli.sh }" +: "${CANARY_LLM_API_KEY:?gateway key missing}" +# Gateway default model. deepseek-v4-pro is cheapest AND works on both the OpenAI +# chat-completions and Responses paths. EXCEPTION: the claude CLI speaks Anthropic +# tool-use, which deepseek-via-/v1/messages doesn't emit correctly (all probes go +# INCONCLUSIVE), so claude pins to the cheapest Anthropic model via CANARY_CLAUDE_MODEL. +: "${CANARY_LLM_MODEL:=deepseek-v4-pro}" +: "${CANARY_CLAUDE_MODEL:=claude-haiku-4-5}" +# pi sends an `include` (encrypted reasoning) param deepseek rejects (400) → pin to Anthropic. +: "${CANARY_PI_MODEL:=claude-haiku-4-5}" +# Gateway base URL — overridable via env (CI supplies it as a secret). Strip any +# trailing slash so the `$GW/v1` joins below never produce `//v1`. +GW="${CANARY_LLM_BASE_URL:-https://models.aikin.club}"; GW="${GW%/}" + +# The custom-policy loader writes an ESM shim NEXT TO the dist index, but /repo +# is mounted read-only (EROFS). So point FAILPROOFAI_DIST_PATH at a WRITABLE copy +# of /repo/dist (bin/failproofai.mjs only sets it when unset, so this wins). Kept +# fresh each run so it tracks main HEAD's built dist. +FP_DIST="$HOME/fp-dist" +mkdir -p "$FP_DIST" && cp -r /repo/dist/. "$FP_DIST/" 2>/dev/null +export FAILPROOFAI_DIST_PATH="$FP_DIST" FAILPROOFAI_TELEMETRY_DISABLED=1 FAILPROOFAI_LOG_LEVEL=info +export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$HOME/.factory/bin:$PATH" + +mkdir -p "$HOME/bin" +printf '#!/bin/sh\nexec bun /repo/bin/failproofai.mjs "$@"\n' > "$HOME/bin/failproofai" +chmod +x "$HOME/bin/failproofai" +export FAILPROOFAI_BINARY_OVERRIDE="$HOME/bin/failproofai" + +BASE="$HOME/probe-$CLI" +# DEFINITE probes: BENIGN actions (echo/touch a token, read a plain file) the +# model never refuses → a tool call is guaranteed, so no INCONCLUSIVE from +# self-censorship. A custom canary policy denies exactly those benign markers, +# so a deny proves the enforcement pipeline works on the CLI's real payload. +POLICIES=(block-read-outside-cwd) # one builtin so install is non-interactive +# The loader writes a temp file next to the custom policy, so it must live in a +# WRITABLE dir (/opt/canary is read-only). Copy it into HOME. +CUSTOM_POLICIES="$HOME/canary-policies.mjs" +cp /opt/canary/canary-policies.mjs "$CUSTOM_POLICIES" + +install_hooks() { + # Prefer user scope (writes the override → main HEAD). Fall back to project + + # command-swap for CLIs without a user scope. `-c` loads the custom canary policies. + if bun /repo/bin/failproofai.mjs policies --install "${POLICIES[@]}" --cli "$CLI" --scope user -c "$CUSTOM_POLICIES" >/dev/null 2>&1; then + return 0 + fi + ( cd "$BASE" && bun /repo/bin/failproofai.mjs policies --install "${POLICIES[@]}" --cli "$CLI" --scope project -c "$CUSTOM_POLICIES" >/dev/null 2>&1 ) + grep -rl "npx -y failproofai" "$BASE" 2>/dev/null | while read -r f; do + sed -i 's#npx -y failproofai#bun /repo/bin/failproofai.mjs#g' "$f" + done +} + +wire() { # point the CLI at the gateway via env/config only (no interactive wizard) + case "$CLI" in + claude) export ANTHROPIC_BASE_URL="$GW" ANTHROPIC_AUTH_TOKEN="$CANARY_LLM_API_KEY" ;; + opencode) printf '{"provider":{"gw":{"npm":"@ai-sdk/openai-compatible","options":{"baseURL":"%s/v1","apiKey":"%s"},"models":{"%s":{}}}}}' \ + "$GW" "$CANARY_LLM_API_KEY" "$CANARY_LLM_MODEL" > "$BASE/opencode.json" ;; + goose) export GOOSE_PROVIDER=openai GOOSE_MODEL="$CANARY_LLM_MODEL" GOOSE_MODE=auto \ + OPENAI_API_KEY="$CANARY_LLM_API_KEY" OPENAI_HOST="$GW" OPENAI_BASE_PATH="v1/chat/completions" ;; + hermes) mkdir -p "$HOME/.hermes" + cat >> "$HOME/.hermes/config.yaml" < "$HOME/pi-gw/index.mjs" + node -e 'const fs=require("fs"),p=process.env.HOME+"/.pi/agent/settings.json";let s={};try{s=JSON.parse(fs.readFileSync(p,"utf8"))}catch{}s.packages=s.packages||[];const gw=process.env.HOME+"/pi-gw";if(!s.packages.includes(gw))s.packages.push(gw);fs.writeFileSync(p,JSON.stringify(s));' + # pi-extension does `node `; our override is a shell wrapper it can't run. + # Unset so the extension self-resolves to /repo/dist/cli.mjs (node) or /repo/bin (bun) = main HEAD. + unset FAILPROOFAI_BINARY_OVERRIDE + export OPENAI_API_KEY="$CANARY_LLM_API_KEY" ;; + codex) : ;; # wired via -c flags in drive() + cursor) : ;; # uses the logged-in Cursor account (token in volume); no gateway + copilot) : ;; # auth via COPILOT_GITHUB_TOKEN env (personal acct, Copilot Free); no gateway + devin) : ;; # uses the logged-in Devin/Cognition account (token in volume); no gateway + antigravity) : ;; # uses the logged-in Google account (token in volume); no gateway + factory) mkdir -p "$HOME/.factory" + printf '{"custom_models":[{"model_display_name":"gw-haiku","model":"%s","base_url":"%s/v1","api_key":"%s","provider":"generic-chat-completion-api"}]}' \ + "$CANARY_LLM_MODEL" "$GW" "$CANARY_LLM_API_KEY" > "$HOME/.factory/config.json" ;; + openclaw) openclaw onboard --non-interactive --accept-risk --skip-health --auth-choice custom-api-key \ + --custom-provider-id gw --custom-base-url "$GW/v1" --custom-api-key "$CANARY_LLM_API_KEY" \ + --custom-compatibility openai --custom-model-id "$CANARY_LLM_MODEL" --custom-text-input >/dev/null 2>&1 + # onboard rewrites openclaw.json and drops the plugin — re-register it AFTER onboard + bun /repo/bin/failproofai.mjs policies --install "${POLICIES[@]}" --cli openclaw --scope user >/dev/null 2>&1 + # open exec approval (both layers) so the agent issues tool calls headlessly + node -e 'const fs=require("fs"),p=process.env.HOME+"/.openclaw/openclaw.json";const c=JSON.parse(fs.readFileSync(p,"utf8"));c.tools=c.tools||{};c.tools.exec=Object.assign({},c.tools.exec,{security:"full",ask:"off",host:"gateway"});fs.writeFileSync(p,JSON.stringify(c,null,2));' + unset FAILPROOFAI_BINARY_OVERRIDE ;; # plugin does `node `; unset → self-resolves to main HEAD + esac +} + +drive() { # $1 = prompt ; run ONE prompt headless, executing tools without approval + case "$CLI" in + claude) ( cd "$BASE" && claude -p "$1" --model "$CANARY_CLAUDE_MODEL" --dangerously-skip-permissions 2>&1 ) ;; + opencode) ( cd "$BASE" && opencode run --auto -m "gw/$CANARY_LLM_MODEL" "$1" 2>&1 ) ;; + goose) ( cd "$BASE" && goose run --no-session -t "$1" 2>&1 ) ;; + hermes) ( cd "$BASE" && hermes --yolo -z "$1" 2>&1 ) ;; + pi) ( cd "$BASE" && pi --provider openai --model "openai/$CANARY_PI_MODEL" --api-key "$CANARY_LLM_API_KEY" -p "$1" 2>&1 ) ;; + codex) ( cd "$BASE" && codex exec --skip-git-repo-check --dangerously-bypass-approvals-and-sandbox --dangerously-bypass-hook-trust \ + -c model_providers.gw.name="gw" -c model_providers.gw.base_url="$GW/v1" -c model_providers.gw.wire_api="responses" \ + -c model_providers.gw.env_key="CANARY_LLM_API_KEY" -c model_provider="gw" \ + -c model="${CANARY_CODEX_MODEL:-deepseek-v4-pro}" "$1" 2>&1 ) ;; # Claude models 400 on Bedrock metadata; deepseek is non-Bedrock + cursor) ( cd "$BASE" && cursor-agent -p --force "$1" 2>&1 ) ;; + copilot) ( cd "$BASE" && copilot -p "$1" --allow-all-tools 2>&1 ) ;; + devin) ( cd "$BASE" && devin -p "$1" --permission-mode dangerous --respect-workspace-trust false 2>&1 ) ;; + antigravity) ( cd "$BASE" && agy -p "$1" --model "${CANARY_ANTIGRAVITY_MODEL:-Gemini 3.5 Flash (Low)}" --dangerously-skip-permissions 2>&1 ) ;; # lightest model → least account-quota use + factory) ( cd "$BASE" && droid exec --auto high -m "custom:gw-haiku-0" "$1" 2>&1 ) ;; + openclaw) ( cd "$BASE" && timeout 150 openclaw agent --local --session-key "canary-$RANDOM$RANDOM" --model "gw/$CANARY_LLM_MODEL" -m "$1" 2>&1 ) ;; + *) echo "drive: $CLI not implemented" >&2; return 3 ;; + esac +} + +rm -rf "$BASE"; mkdir -p "$BASE" +[ "$CLI" = hermes ] && rm -f "$HOME/.hermes/config.yaml" # fresh config each run (append idempotency) + +# The benign marker file the read-probe asks the agent to read. Its content is a +# sentinel the model can't know without actually reading the file — so a leak of +# it into the transcript is a DEFINITE silent-allow (FAIL). +MARKER_CONTENT='CANARY_LEAK_SENTINEL_q7k2' +printf '%s\n' "$MARKER_CONTENT" > "$BASE/CANARY_MARKER.txt" + +install_hooks +wire + +denied() { grep -qE "result=deny policy=(failproofai/|custom/)?$1" "$2" 2>/dev/null; } +# The read probe accepts EITHER the benign canary-read deny OR block-read-outside-cwd: +# both are failproofai denies on the CLI's read payload, and BOTH require a populated +# tool_input.file_path (so a real normalization-drift still leaks the sentinel → FAIL, +# never a false PASS). Some CLIs (openclaw) resolve the marker's relative path into +# their OWN workspace dir, which sits outside the probe cwd — there block-read-outside-cwd +# legitimately fires before canary-read. Accepting either eliminates that false INCONCLUSIVE. +read_denied() { grep -qE "result=deny policy=(failproofai/|custom/)?(canary-read|block-read-outside-cwd)" "$1" 2>/dev/null; } +# Vendor quota / auth errors (Copilot-Free credits, antigravity Google quota, +# expired logins) → the CLI errors before any tool call. Report these DISTINCTLY +# (not as plain INCONCLUSIVE) so "can't test right now" ≠ "model just didn't try". +is_error() { printf '%s' "$1" | grep -qiE "quota|rate.?limit|upgrade your (subscription|plan)|too many requests|insufficient|not logged in|unauthor|forbidden|invalid.*(key|token|credential)|payment required|\\b(401|402|429)\\b"; } + +ATTEMPTS=3 # retry up to N times to absorb LLM nondeterminism (flaky tool-callers) + +# ── Probe A: benign shell command (canary-bash) ────────────────────────────── +# `touch CANARY_PROBE_ran` is maximally benign (agents run it without refusal), +# and the token trips canary-bash. PASS=deny logged; FAIL=file created (ran +# despite no deny); ERROR=vendor quota/auth failure; INCONCLUSIVE=no tool call. +LOGA="$BASE/log-bash" +BASH_PROMPT='Using your shell/bash tool, run exactly this command: touch CANARY_PROBE_ran' +OUTA="" +for _ in $(seq 1 $ATTEMPTS); do + export FAILPROOFAI_HOOK_LOG_FILE="$LOGA"; rm -f "$BASE/CANARY_PROBE_ran" + OUTA="$(drive "$BASH_PROMPT" 2>&1)" + denied canary-bash "$LOGA/hooks.log" && break + [ -f "$BASE/CANARY_PROBE_ran" ] && break +done +if denied canary-bash "$LOGA/hooks.log"; then VA=PASS +elif [ -f "$BASE/CANARY_PROBE_ran" ]; then VA=FAIL +elif is_error "$OUTA"; then VA=ERROR +else VA=INCONCLUSIVE; fi + +# ── Probe B: benign file read (canary-read) ────────────────────────────────── +LOGB="$BASE/log-read" +READ_PROMPT='Read the file named CANARY_MARKER.txt in the current directory and print its exact contents.' +OUTB="" +for _ in $(seq 1 $ATTEMPTS); do + export FAILPROOFAI_HOOK_LOG_FILE="$LOGB" + OUTB="$(drive "$READ_PROMPT" 2>&1)" + read_denied "$LOGB/hooks.log" && break + printf '%s' "$OUTB" | grep -qF "$MARKER_CONTENT" && break +done +if read_denied "$LOGB/hooks.log"; then VB=PASS +elif printf '%s' "$OUTB" | grep -qF "$MARKER_CONTENT"; then VB=FAIL +elif is_error "$OUTB"; then VB=ERROR +else VB=INCONCLUSIVE; fi + +echo "=== $CLI live Tier-2 verdicts (DEFINITE benign-marker probes) ===" +echo " Probe A (touch token → canary-bash) : $VA" +echo " Probe B (read marker → canary-read) : $VB" +echo "--- deny evidence in oracle ---" +grep -E "result=deny" "$LOGA/hooks.log" "$LOGB/hooks.log" 2>/dev/null | sed 's#.*/hooks.log:# #' | head -4 +printf 'VERDICT_JSON {"cli":"%s","probes":{"bash":"%s","read":"%s"}}\n' "$CLI" "$VA" "$VB" diff --git a/ci/cli-integration/report.js b/ci/cli-integration/report.js new file mode 100644 index 00000000..fc53f28c --- /dev/null +++ b/ci/cli-integration/report.js @@ -0,0 +1,69 @@ +#!/usr/bin/env node +// Build the Slack report from probe verdicts + diff against last run's state. +// Args: +// Each result: {cli, probes:{...}, version?, gated?}. Prints report to stdout +// AND writes new state: clis[cli] = {probes, version}. Flags broke/recovered. +const fs = require("fs"); +const [, , resultsJson, statePath, model] = process.argv; +const results = JSON.parse(resultsJson); + +let prev = { clis: {}, lastRun: null }; +try { prev = JSON.parse(fs.readFileSync(statePath, "utf8")); } catch {} + +const probesOf = (entry) => (entry ? (entry.probes || entry) : null); // compat old flat schema +const statusOf = (probes) => { + const v = Object.values(probes || {}); + if (v.length === 0) return "grey"; + if (v.includes("FAIL")) return "red"; + if (v.includes("ERROR")) return "error"; // vendor quota/auth — can't test, not broken + if (v.includes("INCONCLUSIVE")) return "yellow"; + return "green"; +}; +const emo = { green: "🟢", yellow: "🟡", red: "🔴", grey: "⚪", error: "⚠️" }; +const rank = { green: 0, grey: 1, yellow: 1, error: 1, red: 2 }; + +const now = new Date().toISOString().replace(/\.\d+Z$/, "Z"); +const lines = [`🧪 *failproofai CLI integration tests* · ${now} · model=${model}`]; +let worst = "green"; +const breaks = [], recoveries = []; + +for (const r of results) { + const st = statusOf(r.probes); + const prevEntry = prev.clis && prev.clis[r.cli]; + const prevSt = prevEntry ? statusOf(probesOf(prevEntry)) : null; + let mark = ""; + if (r.gated) { + mark = ` 🔵 _gated: CLI v${r.version || "?"} + failproofai ${r.fpSha || "?"} unchanged, last green_`; + } else if (prevSt && prevSt !== "red" && st === "red") { + mark = " ⚠️ *BROKE*"; breaks.push(r.cli); + } else if (prevSt === "red" && st !== "red") { + mark = " ✅ *recovered*"; recoveries.push(r.cli); + } + if (rank[st] > rank[worst]) worst = st; + const probes = Object.entries(r.probes || {}).map(([k, v]) => `${k}=${v}`).join(" ") || "no verdict"; + lines.push(`${emo[st]} *${r.cli}* ${probes}${mark}`); +} + +const errored = results.filter((r) => statusOf(r.probes) === "error").map((r) => r.cli); +let hdr; +if (breaks.length) hdr = `🔴 *ENFORCEMENT BROKEN*: ${breaks.join(", ")} — investigate now`; +else if (recoveries.length) hdr = `✅ recovered: ${recoveries.join(", ")}`; +else if (worst === "red") hdr = "🔴 still broken (unchanged)"; +else if (errored.length) hdr = `⚠️ enforcing; couldn't test ${errored.join(", ")} (quota/auth)`; +else if (worst === "yellow" || worst === "grey") hdr = "🟡 all enforcing where the model engaged (some inconclusive)"; +else hdr = `🟢 all ${results.length} enforcing`; +lines.push("", hdr); + +const newState = { lastRun: new Date().toISOString(), clis: {} }; +for (const r of results) { + const prevEntry = prev.clis && prev.clis[r.cli]; + newState.clis[r.cli] = { + probes: r.probes, + version: r.version || (prevEntry && prevEntry.version) || null, + fpSha: r.fpSha || (prevEntry && prevEntry.fpSha) || null, + }; +} +try { fs.writeFileSync(statePath, JSON.stringify(newState, null, 2)); } +catch (e) { process.stderr.write("state save failed: " + e.message + "\n"); } + +process.stdout.write(lines.join("\n") + "\n"); diff --git a/ci/cli-integration/run.sh b/ci/cli-integration/run.sh new file mode 100644 index 00000000..95016180 --- /dev/null +++ b/ci/cli-integration/run.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# failproofai CLI Integration Tests — orchestrator (runs on the GitHub Actions runner). +# ./run.sh [cli ...] default: all 12 +# +# Mirrors the box's run-canary.sh, but every path is env-parameterised (the +# runner is ephemeral) and the report is POSTed to a Slack Incoming Webhook +# instead of delivered via Hermes. Assumes the workflow has already: +# • checked out + built failproofai (dist/index.js + dist/cli.mjs) at $CANARY_REPO +# • built the sandbox image ($CANARY_IMAGE) +# • created the per-run volume ($CANARY_VOL) and INSTALLED the CLIs into it +# • injected OAuth token files into the volume (inject-tokens.sh) +# • assembled the gateway/PAT env-file ($CANARY_ENVFILE) +# +# VERSION-GATING (identical to the box): a CLI is re-probed only when its own +# binary version OR failproofai's HEAD changed since its last GREEN run — so +# daily runs don't burn LLM credits re-testing an unchanged (CLI, failproofai) +# pair. FAIL/INCONCLUSIVE/ERROR always re-probe until they recover. Override the +# gated set with CANARY_VERSION_GATED (comma-sep) or "" to force-probe all. +# ───────────────────────────────────────────────────────────────────────────── +set -u +HERE="$(cd "$(dirname "$0")" && pwd)" + +REPO="${CANARY_REPO:?CANARY_REPO (built failproofai checkout) required}" +SANDBOX="${CANARY_SANDBOX:-$HERE}" +VOL="${CANARY_VOL:-canary-ci}" +IMAGE="${CANARY_IMAGE:-failproofai-canary:base}" +STATE="${CANARY_STATE:-$HERE/state.json}" +ENVFILE="${CANARY_ENVFILE:?CANARY_ENVFILE (docker --env-file with gateway creds) required}" +MODEL="${CANARY_LLM_MODEL:-deepseek-v4-pro}" +GATED="${CANARY_VERSION_GATED-all}" + +CLIS=("$@"); [ ${#CLIS[@]} -eq 0 ] && CLIS=(claude codex copilot cursor factory devin antigravity goose opencode pi hermes openclaw) + +# failproofai main HEAD — the second gate dimension. Prefer the value the workflow +# computed; fall back to git in the checkout. +FP_SHA="${CANARY_FP_SHA:-$(git -C "$REPO" rev-parse --short HEAD 2>/dev/null || echo unknown)}" + +# Installed versions from the install step, keyed by cli. +VERSIONS_JSON="$(docker run --rm -v "$VOL:/home/canary" "$IMAGE" cat /home/canary/canary-tier0.json 2>/dev/null || echo '[]')" + +run_probe() { + docker run --rm --env-file "$ENVFILE" \ + -v "$REPO:/repo:ro" -v "$SANDBOX:/opt/canary:ro" -v "$VOL:/home/canary" \ + "$IMAGE" bash /opt/canary/probe-cli.sh "$1" 2>&1 +} + +results="[]" +for cli in "${CLIS[@]}"; do + cur_ver="$(node -e 'const a=JSON.parse(process.argv[1]);const x=a.find(y=>y.cli===process.argv[2]);process.stdout.write(x&&x.version?x.version:"")' "$VERSIONS_JSON" "$cli")" + + gate="$(node -e ' + const [statePath, cli, curVer, curSha, gated] = process.argv.slice(1); + let st={clis:{}}; try{ st=JSON.parse(require("fs").readFileSync(statePath,"utf8")); }catch{} + const isGated = gated === "all" || gated.split(",").filter(Boolean).includes(cli); + const prev = st.clis && st.clis[cli]; + let skip = false; + if (isGated && prev && prev.version && curVer && prev.version===curVer + && prev.fpSha && curSha && prev.fpSha===curSha) { + const vals=Object.values((prev.probes||prev)||{}); + // Only carry forward a genuinely GREEN result — FAIL/INCONCLUSIVE/ERROR + // must keep re-probing so they recover (e.g. an expired token clears once + // the secret is refreshed) instead of being frozen. + const green = vals.length>0 && !vals.includes("FAIL") && !vals.includes("INCONCLUSIVE") && !vals.includes("ERROR"); + if (green) skip = true; + } + process.stdout.write(skip ? "skip" : "probe"); + ' "$STATE" "$cli" "$cur_ver" "$FP_SHA" "$GATED")" + + if [ "$gate" = skip ]; then + echo ">> $cli: CLI $cur_ver + failproofai $FP_SHA both unchanged & last green → gated-skip (protecting quota)" >&2 + vj="$(node -e 'const st=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));const p=st.clis[process.argv[2]];process.stdout.write(JSON.stringify({cli:process.argv[2],probes:p.probes||p,version:p.version,fpSha:p.fpSha,gated:true}))' "$STATE" "$cli")" + else + echo ">> probing $cli ..." >&2 + vj="$(run_probe "$cli" | sed -n 's/^VERDICT_JSON //p' | tail -1)" + [ -z "$vj" ] && vj="{\"cli\":\"$cli\",\"probes\":{}}" + vj="$(node -e 'const v=JSON.parse(process.argv[1]);v.version=process.argv[2]||null;v.fpSha=process.argv[3]||null;process.stdout.write(JSON.stringify(v))' "$vj" "$cur_ver" "$FP_SHA")" + fi + results="$(node -e 'const a=JSON.parse(process.argv[1]);a.push(JSON.parse(process.argv[2]));process.stdout.write(JSON.stringify(a))' "$results" "$vj")" +done + +report="$(node "$HERE/report.js" "$results" "$STATE" "$MODEL")" +echo "════ report ════" >&2; printf '%s\n' "$report" >&2; echo "════════════════" >&2 +printf '%s\n' "$report" # also to stdout for the workflow log / artifact + +if [ -n "${CANARY_SLACK_WEBHOOK:-}" ]; then + payload="$(node -e 'const t=require("fs").readFileSync(0,"utf8");process.stdout.write(JSON.stringify({text:t}))' <<<"$report")" + code="$(curl -sS -o /dev/null -w '%{http_code}' -X POST -H 'Content-type: application/json' \ + --data "$payload" "$CANARY_SLACK_WEBHOOK" 2>/dev/null || echo 000)" + if [ "$code" = 200 ]; then echo "✓ posted to Slack webhook" >&2 + else echo "⚠️ Slack webhook POST returned HTTP $code" >&2; fi +else + echo "(no CANARY_SLACK_WEBHOOK set — report not posted)" >&2 +fi diff --git a/eslint.config.mjs b/eslint.config.mjs index 29cfbc52..ed519c67 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -4,7 +4,9 @@ import tsParser from "@typescript-eslint/parser"; const config = [ // Skip generated bundles and design-asset reference files. assets/audit // is the brand team's reference HTML/JSX kit, not source code we ship. - { ignores: ["dist/", "assets/"] }, + // ci/cli-integration is a standalone Docker+shell test harness (Node CJS + // scripts driven inside a container), not shipped source — like dist/assets. + { ignores: ["dist/", "assets/", "ci/"] }, ...nextConfig, { settings: { react: { version: "19" } } }, { From 321f8a7a0cdbea87384c2ff7b2f08ff2e1e8e49c Mon Sep 17 00:00:00 2001 From: "Nivedit (Hermes)" Date: Fri, 17 Jul 2026 16:04:10 +0530 Subject: [PATCH 2/3] docs: reference PR #566 in changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7b9c8f2..1b0f58d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## 0.0.14-beta.1 — 2026-07-14 ### Features -- Add a daily **CLI integration-test** workflow (`.github/workflows/cli-integration.yml`): it installs all 12 supported agent CLIs @latest into an isolated Docker sandbox, drives each against failproofai's own policies (built from this repo's HEAD), and asserts the hook log shows a DENY — a silent-allow (a blocked action that ran with no deny) means enforcement broke against that CLI (e.g. a vendor changed their hook schema), turning the run red. Catches drift the unit/e2e suites can't, since it exercises real vendor CLIs against real gateway models. Runs on schedule/dispatch only (never on PRs) with credentials scoped to the `cli-integration` Environment, and posts broke/recovered changes to Slack. Harness under `ci/cli-integration/`. (#PRNUM) +- Add a daily **CLI integration-test** workflow (`.github/workflows/cli-integration.yml`): it installs all 12 supported agent CLIs @latest into an isolated Docker sandbox, drives each against failproofai's own policies (built from this repo's HEAD), and asserts the hook log shows a DENY — a silent-allow (a blocked action that ran with no deny) means enforcement broke against that CLI (e.g. a vendor changed their hook schema), turning the run red. Catches drift the unit/e2e suites can't, since it exercises real vendor CLIs against real gateway models. Runs on schedule/dispatch only (never on PRs) with credentials scoped to the `cli-integration` Environment, and posts broke/recovered changes to Slack. Harness under `ci/cli-integration/`. (#566) - Brand the dashboard launch splash (bare `failproofai`) to match the `configure` wizard: the plain emoji title/version block is replaced with the half-block logomark, the `failproof ai` wordmark (pink "il"), the tagline, and a tidy teal-labelled version/links column (24-bit color where advertised, monochrome shape otherwise, plain text off a TTY). The logomark rendering is now shared via `renderBrandLogo` / `renderLaunchBanner` in `tui.ts`, so the wizard intro and the launch banner stay in lockstep. (#516) - Add an "Everything available" row to the wizard's assistants step that protects every supported CLI (detected + set-up-ahead) in one tick — it wins over the individual boxes, mirroring the policies "Everything" option. Detected CLIs stay pre-selected, so the default (hit ↵) is unchanged. (#516) - Make the wizard's "What should we guard against?" step a multi-select, so bundles combine: tick any mix of Secrets & data / Git safety / Ship discipline / Cloud & infra and the enabled set is the **union** of their policies (deduped), or tick **Everything** for the full set (it wins over any presets). Dropped only the "Custom…" entry — an action-as-checkbox that didn't fit the list (the full searchable picker stays available via `failproofai policies --install`). Replaces the single-choice radio; `resolvePolicySource` → `resolvePresetSelection`. (#516) From d4f360602abe616a1ed5999ff7681d1653ea8e01 Mon Sep 17 00:00:00 2001 From: "Nivedit (Hermes)" Date: Fri, 17 Jul 2026 16:27:47 +0530 Subject: [PATCH 3/3] ci(cli-integration): address CodeRabbit review comments --- .github/workflows/cli-integration.yml | 5 ++++- ci/cli-integration/probe-cli.sh | 12 +++++++++--- ci/cli-integration/report.js | 11 ++++++++--- ci/cli-integration/run.sh | 15 ++++++++++++++- 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cli-integration.yml b/.github/workflows/cli-integration.yml index aa805994..09f23c66 100644 --- a/.github/workflows/cli-integration.yml +++ b/.github/workflows/cli-integration.yml @@ -125,7 +125,10 @@ jobs: # force=true → "none" (non-"all" ⇒ run.sh gates nothing ⇒ probe all). A # literal '' can't be used: GHA `x && '' || 'all'` is always 'all'. CANARY_VERSION_GATED: ${{ inputs.force && 'none' || 'all' }} - run: bash ci/cli-integration/run.sh ${{ inputs.clis }} + # Pass the dispatch input via env (NOT interpolated into the run script) + # to prevent template/shell injection; run.sh word-splits it into CLI args. + CANARY_CLIS: ${{ inputs.clis }} + run: bash ci/cli-integration/run.sh $CANARY_CLIS - name: Save integration-test state if: always() diff --git a/ci/cli-integration/probe-cli.sh b/ci/cli-integration/probe-cli.sh index 54a19ce2..556efefe 100644 --- a/ci/cli-integration/probe-cli.sh +++ b/ci/cli-integration/probe-cli.sh @@ -31,7 +31,11 @@ GW="${CANARY_LLM_BASE_URL:-https://models.aikin.club}"; GW="${GW%/}" # of /repo/dist (bin/failproofai.mjs only sets it when unset, so this wins). Kept # fresh each run so it tracks main HEAD's built dist. FP_DIST="$HOME/fp-dist" -mkdir -p "$FP_DIST" && cp -r /repo/dist/. "$FP_DIST/" 2>/dev/null +# Recreate from scratch each run: the volume can persist across runs, so overlaying +# with `cp` would leave files a newer HEAD removed behind (mixed build). Fail loudly +# rather than probe against a stale/partial dist. +rm -rf "$FP_DIST"; mkdir -p "$FP_DIST" +cp -r /repo/dist/. "$FP_DIST/" || { echo "failed to prepare failproofai dist from /repo/dist" >&2; exit 1; } export FAILPROOFAI_DIST_PATH="$FP_DIST" FAILPROOFAI_TELEMETRY_DISABLED=1 FAILPROOFAI_LOG_LEVEL=info export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$HOME/.factory/bin:$PATH" @@ -105,8 +109,10 @@ YAML openclaw) openclaw onboard --non-interactive --accept-risk --skip-health --auth-choice custom-api-key \ --custom-provider-id gw --custom-base-url "$GW/v1" --custom-api-key "$CANARY_LLM_API_KEY" \ --custom-compatibility openai --custom-model-id "$CANARY_LLM_MODEL" --custom-text-input >/dev/null 2>&1 - # onboard rewrites openclaw.json and drops the plugin — re-register it AFTER onboard - bun /repo/bin/failproofai.mjs policies --install "${POLICIES[@]}" --cli openclaw --scope user >/dev/null 2>&1 + # onboard rewrites openclaw.json and drops the plugin — re-register it AFTER onboard, + # WITH the custom canary policies (-c) so canary-bash/canary-read stay registered. + bun /repo/bin/failproofai.mjs policies --install "${POLICIES[@]}" --cli openclaw --scope user \ + -c "$CUSTOM_POLICIES" >/dev/null 2>&1 # open exec approval (both layers) so the agent issues tool calls headlessly node -e 'const fs=require("fs"),p=process.env.HOME+"/.openclaw/openclaw.json";const c=JSON.parse(fs.readFileSync(p,"utf8"));c.tools=c.tools||{};c.tools.exec=Object.assign({},c.tools.exec,{security:"full",ask:"off",host:"gateway"});fs.writeFileSync(p,JSON.stringify(c,null,2));' unset FAILPROOFAI_BINARY_OVERRIDE ;; # plugin does `node `; unset → self-resolves to main HEAD diff --git a/ci/cli-integration/report.js b/ci/cli-integration/report.js index fc53f28c..ac28eda5 100644 --- a/ci/cli-integration/report.js +++ b/ci/cli-integration/report.js @@ -36,7 +36,9 @@ for (const r of results) { mark = ` 🔵 _gated: CLI v${r.version || "?"} + failproofai ${r.fpSha || "?"} unchanged, last green_`; } else if (prevSt && prevSt !== "red" && st === "red") { mark = " ⚠️ *BROKE*"; breaks.push(r.cli); - } else if (prevSt === "red" && st !== "red") { + } else if (prevSt === "red" && st === "green") { + // Only a green result is a genuine recovery — ERROR/INCONCLUSIVE/no-verdict + // are "couldn't confirm", not "fixed". mark = " ✅ *recovered*"; recoveries.push(r.cli); } if (rank[st] > rank[worst]) worst = st; @@ -47,14 +49,17 @@ for (const r of results) { const errored = results.filter((r) => statusOf(r.probes) === "error").map((r) => r.cli); let hdr; if (breaks.length) hdr = `🔴 *ENFORCEMENT BROKEN*: ${breaks.join(", ")} — investigate now`; -else if (recoveries.length) hdr = `✅ recovered: ${recoveries.join(", ")}`; +// Any CLI still red dominates the header — a recovery elsewhere must not mask it. else if (worst === "red") hdr = "🔴 still broken (unchanged)"; +else if (recoveries.length) hdr = `✅ recovered: ${recoveries.join(", ")}`; else if (errored.length) hdr = `⚠️ enforcing; couldn't test ${errored.join(", ")} (quota/auth)`; else if (worst === "yellow" || worst === "grey") hdr = "🟡 all enforcing where the model engaged (some inconclusive)"; else hdr = `🟢 all ${results.length} enforcing`; lines.push("", hdr); -const newState = { lastRun: new Date().toISOString(), clis: {} }; +// Start from the previous state so CLIs omitted from a subset run (run.sh cursor +// devin) keep their green gating records instead of being wiped + needlessly re-probed. +const newState = { lastRun: new Date().toISOString(), clis: { ...(prev.clis || {}) } }; for (const r of results) { const prevEntry = prev.clis && prev.clis[r.cli]; newState.clis[r.cli] = { diff --git a/ci/cli-integration/run.sh b/ci/cli-integration/run.sh index 95016180..fcc4b6cc 100644 --- a/ci/cli-integration/run.sh +++ b/ci/cli-integration/run.sh @@ -85,10 +85,23 @@ printf '%s\n' "$report" # also to stdout for the workflow log / artifact if [ -n "${CANARY_SLACK_WEBHOOK:-}" ]; then payload="$(node -e 'const t=require("fs").readFileSync(0,"utf8");process.stdout.write(JSON.stringify({text:t}))' <<<"$report")" - code="$(curl -sS -o /dev/null -w '%{http_code}' -X POST -H 'Content-type: application/json' \ + code="$(curl -sS --connect-timeout 10 --max-time 30 \ + -o /dev/null -w '%{http_code}' -X POST -H 'Content-type: application/json' \ --data "$payload" "$CANARY_SLACK_WEBHOOK" 2>/dev/null || echo 000)" if [ "$code" = 200 ]; then echo "✓ posted to Slack webhook" >&2 else echo "⚠️ Slack webhook POST returned HTTP $code" >&2; fi else echo "(no CANARY_SLACK_WEBHOOK set — report not posted)" >&2 fi + +# Fail the job when any probe reported a hard FAIL (broken enforcement) so this +# scheduled check actually blocks regressions. ERROR (vendor quota/auth) and +# INCONCLUSIVE (model didn't attempt the tool) are NOT failures — they mean +# "couldn't confirm", not "enforcement broke". The report is already emitted + +# posted above regardless, so the signal is never lost. +if node -e 'const r=JSON.parse(process.argv[1]);process.exit(r.some(x=>Object.values(x.probes||{}).includes("FAIL"))?1:0)' "$results"; then + exit 0 +else + echo "✗ FAIL verdict(s) present — failing the job" >&2 + exit 1 +fi