diff --git a/.catalyst-code/skills/add-core-background-task/SKILL.md b/.catalyst-code/skills/add-core-background-task/SKILL.md new file mode 100644 index 0000000..6f89c07 --- /dev/null +++ b/.catalyst-code/skills/add-core-background-task/SKILL.md @@ -0,0 +1,91 @@ +--- +name: add-core-background-task +description: Add a periodic background task to the Rust core (heartbeat, poll, presence publisher) that runs independent of turns +version: 1 +--- + +## When to use + +You need something in `catalyst-code`'s Rust core that runs CONTINUOUSLY, on a +timer, independent of turns — not triggered by a tool call or a user message. +Examples already in the codebase: + +- **`umans_conc` poll** (`main.rs`) — every 5s hits `/v1/usage`, emits + `umans_conc { used, limit, provider }` so the footer shows live concurrency. +- **Presence heartbeat** (`main.rs` + `presence` module) — every 8s publishes + this session's `WorkState` to a per-pid file so peers can detect concurrent + activity, and refreshes a cached peer snapshot. + +If the thing must happen AT a specific point in the turn loop (before/after a +model request, on each tool result), that's a **plugin hook** or a `run_turn` +injection — NOT a background task. Background tasks are for ambient polling/ +broadcasting that has no per-turn trigger. + +## Where things live + +- **Spawn site** — `core/src/main.rs::async fn main()`, AFTER the + `Arc` is constructed and BEFORE `let stdin = …; while let … lines.next_line()`. + The existing `umans_conc` poll block is the canonical template — copy its shape. +- **The task** — a `tokio::spawn(async move { loop { …; tokio::time::sleep(interval).await; } })`. + Capture `state.clone()` (an `Arc`) + any read-once values (pid, a + workspace clone) into the move. Inside, snapshot State fields via their + `tokio::sync::Mutex`/`RwLock` (`.lock().await` / `.read().await`), hold the + guard only as long as needed (`drop(guard)`), then do the I/O. +- **State field for output** — if a consumer needs the polled value on the hot + path (e.g. a per-tool-result check), cache it in a NEW `Mutex` field on + `State` that the task refreshes each tick; consumers read the cache (cheap, + no I/O) instead of re-polling. Add the field to the `State` struct + initialize + it in the `Arc::new(State { … })` literal. (See presence: `peers: Mutex>` + refreshed by the heartbeat, read by `maybe_concurrency_note` in `run_turn`.) + +## Steps + +1. **Decide cadence + whether a hot-path cache is needed.** If a turn-loop + consumer needs the value, add a `State` field for the cached snapshot + (struct decl + init literal) and have the task refresh it each tick. If the + task only EMITS an event (no turn-loop consumer), no State field is needed. +2. **Capture read-once inputs before the move** — `std::process::id()`, a + `workspace.clone()` read from `state.cfg` (cfg is moved into the `RwLock` at + State construction), `home_dir()`, a `started_at` timestamp. These are + constant for the process lifetime, so reading them once avoids per-tick + lock churn. +3. **Do the first write/emit IMMEDIATELY (before the loop)** so a consumer + checking right after startup sees the value — don't wait one full interval. +4. **Spawn the task** mirroring the `umans_conc` block: clone `state`, move it + in, `loop { …work…; tokio::time::sleep(interval).await; }`. Keep work cheap; + a background tick should never block the turn loop (it can't — it's a + separate task — but a slow tick delays the next tick). +5. **Crash-safety for any FILE the task writes** — use atomic temp+fsync+rename + (same pattern as `session`/`memory` persistence). If the task writes a + per-process file, also provide a `clear_()` called on clean shutdown + (the stdin-EOF path at the tail of `main`, after awaiting the running turn) + so the file disappears instantly; tolerate `kill -9` via a stale-reaper on + read (mtime threshold) since the shutdown path doesn't run on SIGKILL. + +## Gotchas + +- **`tokio::time::sleep`, not `std::thread::sleep`.** The task runs on the + async runtime; `std::thread::sleep` blocks a worker thread. +- **Don't hold a `MutexGuard` across `.await`.** Snapshot the value (`.clone()`) + and `drop(guard)` before any `.await`/I/O, or you'll serialize the task behind + the turn loop (and vice versa). The `umans_conc` block does this correctly. +- **`name` vs `&name`** — the dispatch variables in `run_turn` are owned + `String`s; a helper taking `&str` needs `&name` (deref coercion), not `name`. +- **The debug binary `core/target/debug/core` can vanish between + `cargo build` and a smoke run** — a concurrent `cargo test`/`cargo clippy` + (e.g. this session's own harness) reorganizes incremental artifacts and the + main bin can be absent while `deps/` remains. Re-run `cargo build` immediately + before any smoke test that execs the binary. +- **Config root is `~/.config/catalyst-code/`** (post-rename; older notes may + say `umans-harness`). Per-workspace scoping reuses `crate::memory::project_hash` + (already `pub`, fnv1a 16-hex of canonicalized cwd) — no need to promote or + duplicate it. + +## Smoke-testing + +A background task that writes a file is best verified by a real launch: start +the binary with a FIFO stdin, send `{"type":"init"}`, sleep one+ interval, then +check the file exists with the expected content; then close stdin (EOF, NOT +`kill` — `kill` skips the shutdown-cleanup path) and confirm the file is gone. +Use `find ~/.config/catalyst-code// -name "."` to locate the +per-process file (don't assume the workspace-hash subdir — list and match by pid). diff --git a/.catalyst-code/skills/git-commit-all/SKILL.md b/.catalyst-code/skills/git-commit-all/SKILL.md index 4661e08..9217808 100644 --- a/.catalyst-code/skills/git-commit-all/SKILL.md +++ b/.catalyst-code/skills/git-commit-all/SKILL.md @@ -21,16 +21,23 @@ Do NOT use for: ## Steps 1. **Show what will be committed**: `git status --short` and `git diff --stat` so the user can confirm. -2. **Stage everything**: `git add --all` (stages modified, new, and deleted files). -3. **Build a commit message** from the diff summary: - - Read the file list from `git diff --cached --stat` +2. **Verify it compiles** (don't land broken code): run the project's type checker / build for each *changed* component before committing — e.g. `cd core && cargo check` (main binary; avoid `cargo check --tests` if a pre-existing test-binary breakage unrelated to your change is known), `cd tui && go build ./... && go vet ./...`, `cd web && npx tsc --noEmit`. Run the changed ones in parallel; only block the commit on errors in files YOU touched (isolate concurrent-user edits — see the concurrent-user-edits-isolate-errors gotcha). Skip components you didn't touch. + + **Prefer the EXACT CI gates, not just "it compiles"** — especially when the diff touches formatting or adds/changes a CI gate. A plain `cargo check` / `go build` won't catch fmt drift that CI fails on. The authoritative gates (mirror `.github/workflows/ci.yml`): + - core: `cargo fmt --all -- --check` · `cargo clippy --all-targets` (treat warnings as errors under `-D warnings` if CI sets it) · `cargo test --locked` + - tui: `gofmt -l .` (must be EMPTY) · `go vet ./...` · `go build ./...` · `go test -race ./...` (the `-race` matters — catches data-race fixes a plain `go test` won't) + - When a commit ADDS a new CI gate (e.g. a gofmt step), run that gate yourself before committing — the gate's first run should already pass on your commit. +3. **Stage everything**: `git add --all` (stages modified, new, and deleted files). +4. **Build a commit message** from the diff: + - For a SMALL change, `git diff --cached --stat` (file list + churn) is enough. + - For a LARGE or multi-feature diff (hundreds of lines, many files), read the FULL `git diff` (not just `--stat`) — `--stat` won't reveal the distinct features interleaved across shared files like `main.rs`; the actual hunks let you enumerate each feature accurately in the body. - Group by directory/module (e.g. "core: …", "tui: …", "web: …") - Include the primary change type (e.g. "refactor", "fix", "add feature X") - If a single logical change spans files, use one sentence; if multiple, use bullet points - Keep the subject line ≤72 chars; body wraps at 72 -4. **Commit**: `git commit -m ""` -5. **Show the commit**: `git log -1 --oneline` -6. **Push (only if the user asked to push)**: +5. **Commit**: `git commit -m ""` +6. **Show the commit**: `git log -1 --oneline` +7. **Push (only if the user asked to push)**: - Current branch: `git rev-parse --abbrev-ref HEAD` (if it prints `HEAD`, you're in detached HEAD — abort and tell the user to checkout a branch first). - If the branch has an upstream (`git rev-parse --abbrev-ref @{u}` succeeds): `git push`. - If it has NO upstream: `git push -u origin ` (sets upstream on first push). diff --git a/.catalyst-code/skills/production-readiness-review/SKILL.md b/.catalyst-code/skills/production-readiness-review/SKILL.md new file mode 100644 index 0000000..ad55d04 --- /dev/null +++ b/.catalyst-code/skills/production-readiness-review/SKILL.md @@ -0,0 +1,135 @@ +--- +name: production-readiness-review +description: Audit a repo for production readiness before going public — fan out code reviewers + a secrets/PII scan + a build/test gate, then synthesize a go/no-go verdict +--- + +# Production-Readiness Review (Pre-Public Launch) + +Use when the user is about to make a repo **public** (or cut a release) and asks +"is this ready?" / "production readiness" / "can we go public?". Distinct from a +general code-quality review (`parallel-codebase-review`) — this adds the three +things that specifically matter for *going public*: a **secrets/PII audit**, a +**build + test + lint gate**, and an explicit **go/no-go verdict**. + +## When to use +- "Is the repo ready to go public / open-source?" +- Pre-release gate before flipping a repo from private to public. +- "Production readiness review" of a whole codebase. + +## When NOT to use +- Targeted review of one file/feature → read it directly. +- General bug hunt with no public-launch angle → use `parallel-codebase-review`. +- A single component → one `reviewer` suffices, no fan-out needed. + +## Workflow + +### 1. Fan out 3–4 parallel subagents (respect the user's subagent cap) +Each is a fresh context — be self-contained: name the exact files, the focus +areas, the output contract (`file:line` evidence + severity + concrete fix), and +any "ignore this in-progress feature" caveat the user gave. + +| Reviewer | Scope | +|----------|-------| +| `reviewer` (code) | one per major language component (Rust core, Go TUI, …). Adversarial: panics on adversarial input, resource leaks, error swallowing, secret-in-code, security (path confinement, command injection, sandbox). | +| `reviewer` (secrets/PII) | **repo-wide** — hardcoded secrets/tokens/private keys, personal paths/emails/IPs, committed binaries/artifacts, local-only files tracked, LICENSE/README/CI sanity, author fields in manifests. This is the highest-stakes pass. | +| `worker` (build gate) | actually RUN the lint/build/test suite. Report per-step PASS/FAIL + warnings, and split known/in-progress issues from real regressions. | + +Tell each reviewer what is **expected-incomplete** (e.g. "presence feature is +mid-implementation — don't flag its incompleteness") so it doesn't waste budget +on known work, and what's a **known pre-existing issue** vs a new regression. + +### 2. Re-verify the secrets/PII audit YOURSELF — don't trust a truncated summary +A subagent's detailed body sometimes gets truncated to a one-line verdict in the +orchestrator's parallel-task result. For the highest-stakes claim ("secrets are +clean"), **re-run the scan yourself** with `rg --hidden` (rg skips dotfiles by +default — always pass `--hidden`, exclude `.git/`, `target/`, `node_modules`). +Write a small script (don't inline long `rg` chains) and check: + +- Real-looking secrets: `sk-[A-Za-z0-9]{16,}`, `AKIA[0-9A-Z]{16}`, + `gh[pousr]_…`, `xoxb-`, `Bearer `, `BEGIN … PRIVATE KEY`. + Filter out obvious placeholders (`example`, `your_key`, ``, `sk-xxxx`). +- Personal paths: `/home/`, `/Users/`, `/root`, `C:\Users\…` + (a container service user like `harness` is fine; a real username is PII). +- Emails; private keys; internal hostnames (`.local`, `.internal`, cloud-metadata + `169.254.169.254` — note: that IP appearing in code is often the SSRF *risk + being documented*, not a leak). +- Tracked local-only files: `git ls-files | rg '^scripts/|^tmp/|context\.md|plan\.md|\.env|\.log|\.pem|\.key'`. +- Committed large binaries: `git ls-files -z | while read f; do git cat-file -s "HEAD:$f"; done` — flag anything >1MB. +- `LICENSE` + `README` presence; author/owner fields in `Cargo.toml`/`go.mod`/`package.json` (PII leak); CI `secrets.` usage + `pull_request_target` + internal URLs. + +A legitimate MIT/Apache LICENSE attributed to the user's own handle is fine (it's +their copyright, not a leak). The auto-provided `${{ secrets.GITHUB_TOKEN }}` is +standard, not a custom secret. + +### 3. Verify surprising Critical/High findings before reporting them +Re-read the cited `file:line` yourself — converts "the reviewer said" into +"verified." Especially for security claims (e.g. "writes keys world-readable"): +confirm the code actually does what's claimed and that the doc comment (if any) +contradicts it. Line numbers drift; the code is the truth. + +### 4. Synthesize — don't dump +Merge the reports, **dedupe across reviewers**, and rank by severity: +- **P0 blocker** — must fix or do NOT go public (a leaked secret, a crash on + normal input, broken build). +- **P1 should-fix** — small surgical fixes to land before public; not + architectural (a permission gap, a soft wedge, a real data race, fmt drift). +- **P2 nice-to-have** — hardening for a fast follow-up; doesn't block launch. + +Lead with a one-line **go/no-go verdict**, then the P1 table (where / issue / +fix), then P2s, then a **"verified clean"** section so the user knows what WAS +checked (security boundaries, no-panic-on-bad-input, resource bounds, secrets +absent). End by offering to implement the P1 fixes. + +## Gotchas +- **Stale "known issue" memories.** Before treating a compile/test failure as a + known pre-existing issue, actually run the gate — the tree may have moved on + and the issue resolved. Update the stale memory if so (don't leave a wrong + "tests are broken" note that scares off the next session). +- **Subagent body truncation.** If a parallel task returns only a one-line + summary instead of its detailed body, that detail is lost — re-verify the + critical claims yourself rather than reporting the bare verdict. +- **CI blind spots.** The build gate should run the SAME checks CI runs; if CI + omits something (e.g. a Go project whose CI runs `go vet`/`go build`/`go test` + but NOT `gofmt --check`), call that out as a finding — formatting drift slips + through silently. `cargo fmt --all` / `gofmt -w .` are one-command fixes. +- **The subagent hard cap.** If you need >8 parallel reviewers, batch them ≤8 + (the `tasks` mode rejects > `parallel_max_tasks` instantly). See + `parallel-subagent-cap` memory. + +## Applying the findings (fix-all) + +When the user says "fix all issues" / "fix everything found" after a review, +this is the apply-and-verify counterpart. The risk is volume + correctness +across two codebases — structure it so each file tree has ONE writer and the +trickiest fix is yours. + +1. **Delegate a cohesive same-tree cluster to ONE worker.** Group all fixes + that share files (e.g. every `tui/main.go` lifecycle fix: atomic process + ptr, signal-handler quit, startup watchdog, double-`Wait`) into a single + worker so it keeps coherence across its own edits. Give it EXACT fixes + (file:line + the change + the rationale) — workers execute well-specified + edits reliably; vague ones they botch. Tell it what's expected-incomplete + ("presence feature is mid-impl — don't touch it") and that it's the SOLE + writer of its tree (so it may run the formatter itself, no race). +2. **Keep the security-sensitive / judgment-heavy fix for yourself.** SSRF + range-blocking, sandbox-bypass fixes, lifecycle redesigns — read the file, + design carefully, write tests. Don't hand these to a worker. +3. **Verify with the EXACT CI gates, not just "it compiles":** + - core: `cargo fmt --all -- --check` · `cargo clippy --all-targets` · + `cargo test --locked` + - tui: `gofmt -l .` (empty) · `go vet ./...` · `go build ./...` · + `go test -race ./...` (the `-race` matters — it catches the data-race + fixes a plain `go test` won't). +4. **Isolate failures to your own edits.** A test failing in a file you didn't + touch is almost certainly the user's concurrent work (or a pre-existing + issue) — confirm with `git status --short` which modified files are yours + vs. theirs before "fixing" it. (See `concurrent-user-edits-isolate-errors`.) +5. **Spot-check worker output — build-passing ≠ fixes-correct.** A worker + that claims "all 10 fixes done, tests green" may have no-op'd a tricky one. + `grep` for the specific fix markers (e.g. `atomic.Pointer`, `s.busy = false` + in the reset handler, `go fillMentionCache`, `modelIdx = -1`) and read the + one or two trickiest regions yourself. (The `grep` TOOL is flaky — use + `bash grep -n` instead.) +6. **Run the formatters yourself at the end** (`cargo fmt --all`, `gofmt -w .`) + and confirm `--check`/`-l` is clean — this is both a fix (P1 fmt drift) and + the CI gate. diff --git a/.catalyst-code/skills/setup-self-hosted-gh-runner/SKILL.md b/.catalyst-code/skills/setup-self-hosted-gh-runner/SKILL.md new file mode 100644 index 0000000..5fed1da --- /dev/null +++ b/.catalyst-code/skills/setup-self-hosted-gh-runner/SKILL.md @@ -0,0 +1,80 @@ +--- +name: setup-self-hosted-gh-runner +description: Set up a SECURE self-hosted GitHub Actions runner using ephemeral rootless Podman containers (fresh container per job, destroyed after) driven by a lingered systemd user service. Use when GitHub-hosted runners are too slow / too costly and the box runs Linux with Podman. +--- + +# Set up a secure self-hosted GitHub Actions runner (ephemeral rootless Podman) + +## When to use + +- GitHub-hosted runners are slow (cold caches + queue/provisioning) and you have a Linux box with Podman. +- You want ISOLATION: each CI job runs in a fresh container destroyed after the job — no persistence for backdoors, no root daemon, no access to the host's home/secrets/other services. +- Repo can be public or private; rootless Podman + ephemeral containers are secure enough for either (the high-value credential never enters a container). + +Prefer this over bare-metal runners (no isolation) or Docker-based (needs a root daemon). Skip if the box has no Podman (install it + `loginctl enable-linger`), or if jobs need kernel features unavailable in a rootless userns (FUSE mounts, AppImage, certain device access) — keep THOSE jobs on `ubuntu-latest`. + +## Architecture + +``` +systemd user service (gh-runner.service, lingered) + └─ supervisor.sh keeps N ephemeral containers alive + ├─ slot 0: podman run --rm --ephemeral -v gh-cache-0:/cache gh-runner + └─ slot 1: podman run --rm --ephemeral -v gh-cache-1:/cache gh-runner +``` + +- **Ephemeral**: `config.sh --ephemeral` runs exactly ONE job then auto-deregisters; `--rm` destroys the container; the supervisor respawns a fresh one. +- **Rootless**: container `root` (uid 0) is mapped to the unprivileged host user via the user namespace — jobs have no real host privileges. +- **Token hygiene**: supervisor mints a fresh ~1h registration token per spawn via `gh api -X POST`; the high-value `gh` PAT stays on the host and NEVER enters a container. +- **Per-slot cache volumes** (not shared) so concurrent containers don't race on cargo/go caches. Mount ONLY pure caches (GOMODCACHE, GOCACHE, RUNNER_TOOL_CACHE, pip/npm); keep toolchain HOMES (RUSTUP_HOME, CARGO_HOME, /usr/local/go) in the IMAGE so the volume overlay never hides installed tools. + +## Steps + +1. **Recon.** Confirm Podman rootless (`podman info --format '{{.Host.Security.Rootless}}'`), subuid/subgid (`grep $USER /etc/subuid /etc/subgid`), linger (`loginctl show-user $USER | grep Linger` → enable-linger if off), `gh auth status` (needs `repo`+`workflow` scopes), and the repo's CI workflows (`runs-on: ubuntu-latest` jobs to convert). Note any job needing kernel features rootless can't provide (FUSE/AppImage, device access) → leave on ubuntu-latest. + +2. **Clean up prior attempts.** If `~/actions-runner` exists from a stale/misconfigured setup, deregister its offline runner via `gh api -X DELETE repos///actions/runners/` and remove stale `.runner`/`.credentials`. + +3. **Build the runner image** (Containerfile, ubuntu:24.04 base for action compatibility): + - System packages: `build-essential pkg-config libssl-dev ca-certificates git curl jq zip file sudo python3-pip` + the toolchains the repo's CI needs (e.g. Rust via rustup, Go, Node/Bun) + `buildah` if a CI job builds the repo's own Dockerfile. + - Toolchain HOMES in the image (e.g. `ENV RUSTUP_HOME=/opt/rustup CARGO_HOME=/opt/cargo`); point only PURE caches at `/cache` (`ENV GOMODCACHE=/cache/go-mod GOCACHE=/cache/go-build RUNNER_TOOL_CACHE=/cache/toolcache ...`). + - `ADD` the locally-downloaded actions-runner tarball to `/runner` (reuse a downloaded tarball to avoid version guessing). + - See GOTCHAS below — three are MANDATORY in the image or the runner won't start. + +4. **entrypoint.sh**: `config.sh --unattended --url https://github.com// --token $REGISTRATION_TOKEN --name $RUNNER_NAME --labels $LABELS --ephemeral --work _work` then `./run.sh`; `trap './config.sh remove --token $REGISTRATION_TOKEN' EXIT`; `mkdir -p` the /cache subdirs at start (Podman named volumes don't always copy image content on first mount). + +5. **supervisor.sh**: bash loop keeping CONCURRENCY slots alive. Per spawn: `gh api -X POST repos///actions/runners/registration-token --jq .token` (POST!), then `podman run --rm --name -- --memory 7g --memory-swap 0 --cpus 6 --pids-limit 1024 -e REPO/REGISTRATION_TOKEN/RUNNER_NAME/LABELS/RUNNER_ALLOW_RUNASROOT -v gh-cache-:/cache `. On exit, respawn after 5s. A `cleanup_offline()` deletes leaked offline runners for the repo (safe — only your runners are on a private repo). Each slot in its own background `&` loop; `wait` blocks. + +6. **systemd user unit** (`~/.config/systemd/user/gh-runner.service`): `Type=simple`, `Restart=always`, `Environment=` for REPO/CONCURRENCY/IMAGE/LABELS/PATH/HOME, `ExecStart=%h/gh-self-hosted/supervisor.sh`, `WantedBy=default.target`. `systemctl --user daemon-reload && systemctl --user enable --now gh-runner`. + +7. **Convert the repo's CI workflows**: `runs-on: ubuntu-latest` → `runs-on: [self-hosted, Linux, X64]`. If a job uses `docker/buildx-action` (needs Docker), convert to `buildah build --isolation chroot --storage-driver vfs -t .` (works rootless-in-rootless). Commit on a branch + open a PR (pull_request triggers CI without touching master/release workflows). + +8. **Verify end-to-end**: confirm runners `online` + `busy:false` via `gh api repos///actions/runners`; watch a real job land (`journalctl --user -u gh-runner` shows "Running job:" / "completed with result: Succeeded"). Expect the FIRST run to be cold (caches empty); subsequent runs fast. + +## GOTCHAS (all MANDATORY — the runner will NOT start without these) + +1. **Runner refuses root: "Must not run with sudo".** Rootless Podman maps host user → container root (uid 0); the runner bails on uid 0. FIX: env `RUNNER_ALLOW_RUNASROOT=1` (pass via `podman run -e`; bake at the END of the Containerfile so it doesn't invalidate cached layers). + +2. **.NET listener crashes (exit 134): "Couldn't find a valid ICU package".** The runner listener is a .NET 6 app. FIX: `RUN cd /runner && ./bin/installdependencies.sh` in the image (installs libicu + liblttng-ust + krb5). The "Execute sudo ./bin/installdependencies.sh" message is FATAL, not a warning. + +3. **buildah can't resolve short-name images: "short-name 'rust:1.82-slim' did not resolve... no unqualified-search registries".** FIX: write `/etc/containers/registries.conf` with `unqualified-search-registries = ["docker.io"]`. + +## Other gotchas + +4. **`systemctl --user` from a non-login shell (agent bash, cron) fails**: "$DBUS_SESSION_BUS_ADDRESS and $XDG_RUNTIME_DIR not defined". FIX: `export XDG_RUNTIME_DIR=/run/user/$(id -u) DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$(id -u)/bus` first. Requires `loginctl enable-linger `. + +5. **Registration token endpoint is POST.** `gh api repos///actions/runners/registration-token` defaults to GET. Use `gh api -X POST ... --jq .token`. Tokens live ~1h; mint fresh per spawn. + +6. **Orphaned-job wedge**: killing a runner container mid-job leaves the job "in_progress" on an offline runner → blocks runner delete (422) AND blocks `gh run rerun` ("already running"). Recover: `gh run cancel `, then trigger fresh (empty commit → new pull_request run); GitHub eventually clears it. Ephemeral + `--rm` + a `config.sh remove` trap prevents it in normal operation. + +7. **Per-slot cache volumes, not shared.** Concurrent containers on the SAME volume race on cargo-registry/go-mod writes. Give each slot its own volume. + +8. **Runner auto-update is fine in ephemeral mode** (downloads newer version, restarts within the container, completes cleanly). A brief "Not configured" crash-loop right after a supervisor restart is a TRANSIENT race (old containers tearing down + a concurrent rerun); it self-resolves — watch for 0 errors over ~20s rather than reacting to the first burst. + +## Manage + +```bash +systemctl --user status|restart|stop gh-runner +journalctl --user -u gh-runner -f # supervisor + runner logs +gh api repos///actions/runners --jq '.runners[]|{name,status,busy}' +# rebuild image: cd ~/gh-self-hosted && podman build -t gh-runner . && systemctl --user restart gh-runner +# tune: systemctl --user edit gh-runner (Environment=CONCURRENCY=3 ...) +``` diff --git a/.catalyst-code/skills/ttft-prompt-caching-audit/SKILL.md b/.catalyst-code/skills/ttft-prompt-caching-audit/SKILL.md new file mode 100644 index 0000000..a60e6d4 --- /dev/null +++ b/.catalyst-code/skills/ttft-prompt-caching-audit/SKILL.md @@ -0,0 +1,98 @@ +--- +name: ttft-prompt-caching-audit +description: Audit the request path for Time-to-First-Token health — prefix-cache stability, provider caching opt-in, and the already-captured cache-hit metric +version: 1 +--- + +## When to use + +Use this when the user asks about TTFT / latency / "first token" speed, or when turns +feel slow to start responding. TTFT in an LLM agent harness is dominated by two things: +**prefix-cache hit rate** (how much of the prompt the provider can reuse from the prior +turn) and **request size**. This audit walks the code paths that decide both. + +The audit is grounded in THIS harness's provider path (`core/src/`): `provider.rs` +(`stream_turn_*`), `message.rs::build_anthropic_request`, `main.rs` `run_turn`, +`logging.rs::grounded_estimate`, `config.rs` compaction knobs. + +## Steps + +1. **Confirm streaming is on.** Each `stream_turn_*` sets `stream: true`. If a path + doesn't, that's the first bug — non-streaming means waiting for the full response + before any token. + +2. **Check system-prompt stability (the cacheable prefix).** Read + `build_system_prompt` (main.rs ~87). It must be assembled from STABLE sources + (constants, git context, memory, plugin docs, skill manifest) and NOT mutate per + turn. Anything injected that changes every turn (a timestamp, a per-request id) + at the *head* busts the entire prefix cache. + +3. **Check the rolling work-state / context-summary placement.** Find where the + transient summary is pushed (`work_state_message`, main.rs ~3844). It MUST be a + TAIL message (last in the array) and popped before persisting (main.rs ~3867), so + updating it never invalidates the cached prefix. If it's spliced into the system + prompt or mid-stream, it busts the cache every turn. + +4. **Check the sanitizer's no-op behavior.** `sanitize_orphaned_tool_calls` + + `sanitize_tool_call_arguments` run unconditionally before every request, but must + only rewrite+persist when they actually changed something (main.rs ~3790). On a + clean turn it's an O(n) scan that returns 0 — that's fine. When it fires (rare: + aborted turn, malformed args) it rewrites history → one busted turn (unavoidable). + +5. **Check compaction/digest tuning (request size).** `context_compact_at` (default + 0.90) and `context_digest_at` (default 0.40, config.rs ~730). The soft digest at + 40% is GOOD for TTFT — it collapses stale large tool results into one-line digests + well before compaction, shrinking every subsequent request. If `context_digest_at` + is 0 or very high, request size (and thus TTFT) creeps up over a long session. + +6. **THE KEY STEP — check per-PROVIDER caching opt-in.** This is where most TTFT + wins hide. Grep the tree for `cache_control`: + - **OpenAI-compatible path** (Umans/GLM/Qwen, `stream_turn_openai`): caching is + IMPLICIT — no opt-in needed. Already captured via `cached_tokens` from + `prompt_tokens_details.cached_tokens`. Nothing to add; the lever is measurement + (step 7). + - **Anthropic/Claude path** (`stream_turn_anthropic` → `message.rs::build_anthropic_request`): + Anthropic does NOT cache by default. You MUST set `cache_control` either as a + top-level field (automatic) or as explicit breakpoints on content blocks. If + `cache_control` appears nowhere in the Anthropic builder, every Claude turn + reprocesses the full prompt and `cache_read_input_tokens` comes back 0 — the + single biggest TTFT lever. + - **Anthropic subtlety that defeats the naive fix:** automatic caching puts the + breakpoint on the LAST block. Our last block is the transient work-state tail, + which changes every turn → zero hits (Anthropic's docs flag this exact pattern + as the "common mistake"). So use EXPLICIT breakpoints: on the last system block + (always hits within a session; the system prompt is large enough to clear the + 1024–4096-token min threshold) and a rolling one on the last PERSISTED message + (the 20-block lookback then gives per-turn hits, since each turn adds <20 + blocks). Keep breakpoints OFF the changing work-state tail. Verify against the + live Anthropic prompt-caching docs (fetch https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching) + since the API and breakpoint rules evolve. + +7. **Measure with what's already captured.** Both provider paths already report + cache tokens: OpenAI `cached_tokens` (parsed in `stream_turn_openai`) and + Anthropic `cache_read_input_tokens` (parsed in `stream_turn_anthropic`). They're + logged in `turn_done` (main.rs ~4452) alongside `ttft_ms`. The diagnostic, zero + code: if `cached_tokens ≈ tokens_in`, caching is healthy; if it's ~0, something + is busting the prefix — re-walk steps 2–6 to find it. Surfacing the cache-hit ratio + (`cached_tokens / tokens_in`) next to `ttft_ms` in the UI is the cheapest way to + make cache health visible instead of buried in logs. + +## Answer shape + +- Lead with what's already correct (so the user knows the foundation is sound). +- Name the ONE concrete gap with a file reference and why it matters. +- Give the prioritized fix (usually: add explicit `cache_control` breakpoints to the + Anthropic path; surface the cache-hit metric) and offer to implement. +- Note the provider split: OpenAI/Umans is implicit-cached (fine); Anthropic needs + explicit opt-in (the gap). + +## Avoid + +- Don't claim a provider "auto-caches" without verifying against its current docs — + OpenAI does implicit prefix caching; Anthropic requires explicit `cache_control`. + These rules change; fetch the docs to be precise before advising. +- Don't put a cache breakpoint on content that changes every turn (timestamps, + per-request context, the transient work-state tail). The lookback only finds + entries WRITTEN at breakpoints; a changing breakpoint yields zero hits. +- Don't conflate request-size levers (compaction/digest) with cache-hit levers — + both affect TTFT but are independent. Tune both. diff --git a/.catalyst-code/skills/wire-tui-blocking-flyout/SKILL.md b/.catalyst-code/skills/wire-tui-blocking-flyout/SKILL.md new file mode 100644 index 0000000..b955137 --- /dev/null +++ b/.catalyst-code/skills/wire-tui-blocking-flyout/SKILL.md @@ -0,0 +1,181 @@ +--- +name: wire-tui-blocking-flyout +description: Surface a core wire event as a blocking flyout/modal in the Go TUI (event case + key dispatch + render overlay). Use when the core emits a blocking-prompt event (ask/approval/intercom-style) that the TUI must render and capture input for. +--- + +# Wire a core wire event → Go TUI blocking flyout + +Use when the Rust core emits a wire event that should pop a **blocking flyout / +modal** in the Go TUI and capture user input until resolved (the core blocks on +a `Notify` until the TUI sends a reply command). Existing instances: +`ask_request` (ask tool), `approval_request` (approval gate), `intercom_message` +(subagent need_decision). The shape is identical for all three. + +This is the **TUI counterpart** of `core-event-to-web` (which covers the Next.js +web side) and is NOT the same as `add-tui-tool-renderer` (which renders a tool +*block* in the transcript — non-blocking, no input capture). + +## When to use + +- A core event needs a modal/flyout the user interacts with (select, type, submit). +- The core blocks until the TUI sends a reply command (`ask_reply`, `approve`, + `intercom_reply`). +- NOT for: passive transcript rendering (→ `add-tui-tool-renderer`), web-only + surfacing (→ `core-event-to-web`), or adding the core-side blocking machinery + (→ `add-blocking-tool`). + +## The three integration points (ALL required — missing any = dead code) + +The TUI keeps per-prompt state on the `session` struct (e.g. +`pendingAsk *askPrompt`, `pendingApproval *approvalPrompt`). A prompt type is +dead code unless ALL THREE of these wire it in: + +### 1. Event handler — `tui/handlers.go` `handleCoreEvent` switch +Add a `case "":` that parses the event payload into the prompt +struct and assigns it to the session field. Place it near the other blocking +prompts (`approval_request` / `intercom_message`). + +```go +case "ask_request": + // rawKey is REQUIRED for structured fields (arrays/objects): ev.get + // unmarshals into a string, which FAILS for an array → returns "". + qraw, ok := ev.rawKey("questions") + if !ok { + qraw = json.RawMessage("[]") + } + if a := parseAskRequest(ev.get("request_id"), qraw); a != nil { + s.pendingAsk = a + s.input.Blur() // modal owns keys; don't leave the chat input blinking + s.logInfo("❓ agent asks …") // transcript marker (approval/intercom both log) + s.layout() + } +``` + +### 2. Key dispatch — `tui/handlers.go` `handleKey` +Add an intercept right AFTER the modal intercept, BEFORE scroll/global keys. A +blocking flyout owns all keys (option cycling, text entry, submit, skip): + +```go +if s.modal.kind != modalNone { + return s.handleModalKey(msg) +} +// ↓ insert here — same precedence as a modal +if s.pendingAsk != nil { + return s.handleAskKey(msg) +} +``` + +Without this the prompt state is set but receives zero keystrokes — the user +can't answer and the core blocks forever. + +### 3. Render — `tui/render.go` `View()` +Apply the overlay at the END of `View()`, after the modal overlay. The overlay +helper is a no-op (returns `base` unchanged) when nothing is pending, so it can +be called unconditionally: + +```go +view := strings.Join(parts, "\n") +if s.modal.kind != modalNone { + view = s.renderModalOverlay(view) // was: return s.renderModalOverlay(view) +} +return s.renderAskOverlay(view) // no-op when s.pendingAsk == nil +``` + +The overlay uses `lipgloss.Place(w, h, Center, Center, box)` — it centers the +box in a w×h field of spaces, blanking the background (same as `renderModalOverlay`). + +## The rawKey-vs-get gotcha (the #1 silent failure) + +`coreEvent.get(key)` unmarshals the value into a **string** — which FAILS for a +JSON array/object and returns `""`. So `ev.get("questions")` on an +`ask_request` (whose `questions` is an array) yields `""`, the parser gets empty +input, and the flyout never opens. **Always use `ev.rawKey(key)`** (returns +`json.RawMessage`) for any structured field. This is the exact bug that left the +TUI's ask feature dead for its entire existence. + +## Validation errors: transient inline, not transcript spam + +A blocking flyout's submit-failure (e.g. an empty required field) must set an +`errMsg` field on the prompt struct and render it **inside the flyout box** +(cleared on the next non-submit keypress), NOT call `s.logError(...)`. The +latter appends a permanent "✗ …" line to the transcript on EVERY Enter — a +user mashing Enter on an empty required field spams the log (observed 6× in the +wild). Mirror the `intercomNudge` pulse pattern, not the transcript log. + +## Key handling: action names + hardcoded fallbacks + +Two gotchas that left the ask flyout's navigation dead even after wiring: + +1. **`s.kb(msg, action)` silently returns false for unregistered action names.** + The keybind registry (`keybindDefs` in `tui/keybinds.go`) is the single + source of truth for action names. `s.kb(msg, "next_field")` returns false + because the registered action is `"field_next"` (tab). The ask code used + invented names (`next_field`/`down`/`prev_field`/`up`) — none matched, so + navigation never fired. Always grep `keybindDefs` for the exact Action string + before writing `s.kb`/`s.kbAny` calls. Common ones: `field_next`/`field_prev` + (tab/shift+tab), `nav_down`/`nav_up` (↓/↑), `nav_down_alt`/`nav_up_alt` + (j/k), `send` (enter), `close` (esc), `cycle_left`/`cycle_right` (←/→/h/l). + +2. **Blocking flyouts need hardcoded `msg.String()` arrow fallbacks**, mirroring + the scroll handler (`msg.String() == "up" || s.kbAny(msg, "nav_up", + "nav_up_alt")`). Relying solely on the keybind map means a user who + disabled/rebound a nav key in `/keybinds` can't navigate the flyout at all. + Arrows must ALWAYS work for a blocking prompt: + ```go + if s.kb(msg, "field_next") || msg.String() == "down" || s.kbAny(msg, "nav_down", "nav_down_alt") { + ``` + +## Verify + +- `cd tui && go build ./...` — must pass. +- `cd tui && go vet ./...` — must pass. +- `cd tui && go test ./...` — must pass. +- `gofmt -l .go` — empty output = clean (CI runs `go vet`/`go test`/`go + build` for the TUI, not `gofmt`, but keep changed files formatted). + +## Test pattern + +Exercise the REAL event path via `handleCoreEvent` with a constructed +`*coreEvent`, not just direct field assignment — that's what guards the event +wiring (the part that was missing): + +```go +func askRequestEvent(t *testing.T, requestID, questions string) *coreEvent { + raw, _ := json.Marshal(map[string]any{ + "request_id": requestID, + "questions": json.RawMessage(questions), + }) + return &coreEvent{Type: "ask_request", Raw: raw} +} + +func TestAskRequestSetsFlyout(t *testing.T) { + s := initialSession() + s.ready = true + s.width, s.height = 80, 24 + s.layout() + s.handleCoreEvent(askRequestEvent(t, "ask-1", `[{"id":"x","prompt":"X?","type":"select","options":["A","B"],"required":true}]`)) + if s.pendingAsk == nil { t.Fatal("ask_request must set pendingAsk") } +} +``` + +Also test: render produces the prompt text (`stripANSI(s.renderAskOverlay(base))` +contains the question), Enter submits + clears, Esc skips + clears. + +**Test isolation gotcha:** `initialSession()` calls `loadSettings()`, which +reads the user's REAL `~/.config/catalyst-code/settings.json` — so any test that +depends on keybinds (e.g. asserting `k` navigates via `nav_up_alt`) is +environment-dependent and will fail on a machine where the user disabled that +binding. Reset to defaults in keybind-sensitive tests: +```go +s := initialSession() +s.keybinds = defaultKeybinds() // isolate from user settings +``` + +## Diagnostic methodology (don't chase exotic races) + +When "tool/event not surfacing in a frontend" is reported, **grep BOTH +frontends for the event case + dispatch + render wiring FIRST**, before +theorizing about restart races, field-name mismatches, or ordering bugs. The +common cause is a missing integration point (dead code), not a subtle race. The +`ask-tool-restart-wedge` memory was a red herring for a report that was simply +the TUI never handling `ask_request` at all. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96618fd..37bf717 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ permissions: jobs: core: name: core (rust) - runs-on: ubuntu-latest + runs-on: [self-hosted, Linux, X64] steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable @@ -32,7 +32,7 @@ jobs: tui: name: tui (go) - runs-on: ubuntu-latest + runs-on: [self-hosted, Linux, X64] steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 @@ -45,6 +45,9 @@ jobs: ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('tui/go.sum') }} restore-keys: ${{ runner.os }}-go- + - name: gofmt check + working-directory: tui + run: test -z "$(gofmt -l .)" - name: go vet working-directory: tui run: go vet ./... @@ -57,7 +60,7 @@ jobs: cross-compile: name: tui cross-compile (${{ matrix.goos }}/${{ matrix.goarch }}) - runs-on: ubuntu-latest + runs-on: [self-hosted, Linux, X64] strategy: fail-fast: false matrix: @@ -89,14 +92,10 @@ jobs: docker: name: docker image - runs-on: ubuntu-latest + runs-on: [self-hosted, Linux, X64] steps: - uses: actions/checkout@v4 - - uses: docker/setup-buildx-action@v3 - - name: build (no push) - uses: docker/build-push-action@v6 - with: - context: . - push: false - load: true - tags: catalyst-code:ci + - name: build image (no push; buildah chroot — runs under rootless podman) + # Nested OCI build: chroot isolation + vfs storage work inside a + # rootless container without extra privileges (build-check only). + run: buildah build --isolation chroot --storage-driver vfs -t catalyst-code:ci . diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..621b4c1 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,65 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best for the overall community, not just ourselves + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances + of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing the standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies +when an individual is officially representing the community in public spaces. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the maintainers responsible for enforcement. All complaints will be +reviewed and investigated promptly and fairly. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +https://www.contributor-covenant.org/version/2/1/code_of_conduct.html. + +[homepage]: https://www.contributor-covenant.org + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1f33f12 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,90 @@ +# Contributing to Catalyst Code + +Thanks for your interest in contributing! Catalyst Code is a coding-agent +harness made of four cooperating components around a single stdio JSONL +protocol. + +## Architecture at a glance + +| Component | Language | Role | +|-----------|----------|------| +| `core/` | Rust (tokio) | The engine: conversation, model streaming, tools, sessions, plugins, subagents | +| `tui/` | Go (Bubble Tea) | Terminal UI; spawns `core` and speaks JSONL | +| `sdk/` | TypeScript | Thin pi-compatible wrapper so pi-web can swap in the harness | +| `web/` | Next.js 15 + React 19 | Web equivalent of the TUI | + +## Getting started + +Build the core and TUI from the repo root: + +```sh +./build.sh # cargo build --release (core) + go build (tui) +``` + +Or individually: + +```sh +cargo build --release --manifest-path core/Cargo.toml # → core/target/release/core +cd tui && go build -o catcode . # → tui/catcode +``` + +The web frontend has its own setup (`cd web && bun install && bun run dev`). + +## Before opening a PR + +**Core (Rust):** + +```sh +cd core +cargo fmt --all -- --check +cargo clippy --all-targets +cargo test --locked +``` + +**TUI (Go):** + +```sh +cd tui +gofmt -l . # must be empty +go vet ./... +go test ./... +go build ./... +``` + +CI runs all of the above, so run them locally first to save a round-trip. + +## Code style + +- **Rust:** `cargo fmt` (rustfmt default). Avoid `unwrap()`/`expect()` on data + that comes from the model, files, or the network — prefer `?`/`unwrap_or`/ + explicit error returns so adversarial input can't crash the core. +- **Go:** `gofmt -s`. The TUI is single-threaded `Update` + channel-only + goroutines; keep shared state behind the `session` model and communicate via + channels/`tea.Cmd`, not shared mutable globals. + +## Security notes for contributors + +- File tools confine paths to the workspace (`..`/absolute/symlink escapes are + rejected). Don't add a path-handling tool that bypasses `workspace::resolve`. +- The `bash` tool runs under an optional sandbox (`--sandbox firejail`, + `--no-network`). Treat the denylist as a tripwire, not a sandbox. +- Secrets: never log API keys or OAuth tokens. The `set_key` path logs only the + provider name, never the key. Config files holding keys are written `0600`. +- Plugins from a repo's `.catalyst-code/plugins/` load only with an explicit + `--trust-project-plugins` opt-in — never read that flag from a config file a + repo could ship. + +## Commit messages + +Use a short, imperative subject (`add fetch SSRF hardening`, not `added`). +Reference the issue/PR number in the body when relevant. + +## Reporting security issues + +Please do **not** open a public issue for security vulnerabilities. See +`SECURITY.md` if present, or contact the maintainers privately. + +## License + +By contributing, you agree your contributions are licensed under the MIT +License (see `LICENSE`). diff --git a/README.md b/README.md index da183e0..8ac4089 100644 --- a/README.md +++ b/README.md @@ -1,103 +1,148 @@ -

- Catalyst Code logo -

+ + +[![Contributors][contributors-shield]][contributors-url] +[![Forks][forks-shield]][forks-url] +[![Stargazers][stars-shield]][stars-url] +[![Issues][issues-shield]][issues-url] +[![MIT License][license-shield]][license-url] +[![Website][website-shield]][website-url] + +
+
+ + Catalyst Code logo + + +

Catalyst Code

+ +

+ A self-hosted, OpenAI-compatible coding-agent harness — one binary, any provider, with a human-in-the-loop approval gate. +
+ Releases » +
+
+ View Demo + · + Report Bug + · + Request Feature +

+
-

Catalyst Code

+
+ Table of Contents +
    +
  1. + About The Project + +
  2. +
  3. + Getting Started + +
  4. +
  5. Usage
  6. +
  7. Providers and Login
  8. +
  9. Architecture
  10. +
  11. Subagents and Intercom
  12. +
  13. Roadmap
  14. +
  15. Contributing
  16. +
  17. License
  18. +
  19. Contact
  20. +
  21. Acknowledgments
  22. +
+
-

- A production-grade, OpenAI-compatible coding-agent harness.
- Native multi-provider — Umans · OpenAI · Gemini · Anthropic — with a human-in-the-loop approval gate. -

+## About The Project + +Catalyst Code is a coding agent you run on your own machine against **any +OpenAI- or Anthropic-compatible endpoint** — a cloud API, a local model, or a +self-hosted gateway. It is not a hosted service and does not phone home: your +code, prompts, and API key stay on your box. + +It exists because most agent tooling is locked to one provider, runs as a hosted +SaaS, or ships as an opaque binary. Catalyst Code is the opposite — a small set +of readable components sharing one newline-delimited JSON protocol, with a real +safety model (workspace confinement + an approval gate) and first-class +subagents. You can drive it from the terminal, the browser, or your own code. + +Key highlights: + +* **Multi-provider, no lock-in** — one `/login` picker for Umans, OpenAI, Gemini, + and Anthropic. Be logged into several at once; any model you pick routes that + turn to its endpoint. API key *and* subscription OAuth (no key) supported. +* **Human-in-the-loop safety** — destructive tools (`bash`, `write_file`, + `edit`, …) require consent under the default `destructive` mode. Restricted + paths (`.env`, `.git`, `.ssh`) are gated for reads *and* writes. Optional Linux + hard sandbox: `--sandbox firejail --no-network`. +* **Workspace confinement** — every file op resolves against a workspace root; + absolute paths, `..`, and symlink escapes are rejected. `bash` runs with + `cwd = workspace`. +* **Built-in subagents** — delegate to focused child agents (`scout`, + `planner`, `worker`, `reviewer`, …) over single / parallel / chain execution, + with a peer intercom bus for coordination. +* **Robust by default** — HTTP retry/backoff, idle-stream timeout, summarizing + context compaction with orphaned-tool-call sanitization, fsync'd append-only + sessions, and core-crash auto-recovery. +* **Pluggable** — bundled agents, skills, and hook-based plugins (pre/post + + lifecycle + `pre_turn` model handoff) live in [`.catalyst-code/`](.catalyst-code). + Plugins can even declare custom tools without MCP or a recompile. +* **Full toolset** — `edit`/`patch`/`grep`/`glob`, async `bash`, `diagnostics` + (cargo / tsc / go / py), `fetch`, `todo`, `memory`, git tools, `spawn`, and + `subagent` delegation. -

- version - platforms - Rust - Go - license -

+

(back to top)

---- +### Built With -## Overview +* [![Rust][Rust-badge]][Rust-url] — async engine (`core/`) +* [![Go][Go-badge]][Go-url] — terminal UI (`tui/`, Bubble Tea) +* [![Next.js][Next.js-badge]][Next.js-url] — web frontend (`web/`, React 19) +* [![TypeScript][TypeScript-badge]][TypeScript-url] — SDK wrapper (`sdk/`) -Catalyst Code is a self-hosted coding agent that runs against any OpenAI- or -Anthropic-compatible endpoint. Four cooperating components share one -newline-delimited JSON protocol over stdio: +

(back to top)

-| Component | Language | Role | -|:---|:---|:---| -| **`core/`** | Rust (async, tokio) | The engine — conversation, model streaming, an agentic tool loop with a human-in-the-loop approval gate, sessions, memory, plugins, and subagents. | -| **`tui/`** | Go · [Bubble Tea](https://github.com/charmbracelet/bubbletea) | The terminal interface (`catcode`). Spawns the core, streams events, renders approvals and metrics. | -| **`sdk/`** | TypeScript | A thin pi-compatible wrapper (`@catalyst-code/coding-agent`) so the web frontend can drive the core. | -| **`web/`** | Next.js 15 · React 19 | The browser equivalent of the TUI — an SSE bridge to one core process. | - -> **v0.2.0** ships the production hardening layer: subagents + intercom, -> summarizing context compaction, session token budgets, `--sandbox firejail` -> + `--no-network`, persistent per-workspace sessions, vision input, core-crash -> auto-recovery, multi-provider `/login` (API key **and** OAuth), a Dockerfile, -> and cross-platform install scripts. Full history in -> [`CHANGELOG.md`](CHANGELOG.md). - ---- - -## Table of contents - -- [Installation](#installation) - - [Linux & macOS — `install.sh` (recommended)](#linux--macos--installsh-recommended) - - [Windows — MSI + `install-web.ps1`](#windows--msi--install-webps1) - - [First run](#first-run) - - [Web frontend (as a service)](#web-frontend-as-a-service) - - [Prebuilt binaries (standalone)](#prebuilt-binaries-standalone) -- [Features](#features) -- [Providers and login](#providers-and-login) -- [Build from source](#build-from-source) -- [Architecture](#architecture) -- [Subagents and intercom](#subagents-and-intercom) -- [Releases](#releases) -- [Protocol](#protocol) -- [Testing](#testing) -- [Security and notes](#security-and-notes) -- [License](#license) - ---- - -## Installation - -The **recommended** path is the bundled install script — it **downloads -prebuilt binaries** (no compiler needed) and installs `catcode` + -`catcode-core` to your PATH, and can also install the web frontend as a -background service. Pass `--build-from-source` to compile locally instead. -Standalone no-install binaries (AppImage / `.dmg` / MSI) are also -[available](#prebuilt-binaries-standalone). - -> **Repo visibility:** the installer downloads from this repo's GitHub -> Releases. Anonymous download works once the repo is **public**. If it is -> private, point the installer at a public mirror with `--base-url ` -> (Linux/macOS) / `-BaseUrl ` (Windows), or build from source. - -### Linux & macOS — `install.sh` (recommended) - -`install.sh` **downloads prebuilt** `catcode` + `catcode-core` and installs -them to your PATH — no Rust/Go/`next build` on the host. With `--with-web` it -also downloads a prebuilt Next.js web bundle and installs it as a system -service (**systemd** on Linux, **launchd** on macOS — auto-detected). The web -service only needs a [Node](https://nodejs.org) or [Bun](https://bun.sh) -runtime to *run* (not to build). - -**Prerequisites:** `curl` + `coreutils` (always); Node or Bun (only for -`--with-web`). No compiler. Add `--build-from-source` to compile locally -(then also needs Rust + Go 1.24.2+). +## Getting Started + +Get up and running in under a minute — no clone, no compiler. + +### Prerequisites + +* **Linux / macOS / Windows** (the hard sandbox is Linux-only; on macOS/Windows + leave it `none`). +* `curl` + coreutils (always). No compiler unless you build from source. +* For the web frontend: a [Node](https://nodejs.org) or [Bun](https://bun.sh) + runtime to *run* the service (not to build it). +* To build from source: Rust (stable) + **Go 1.24+**. + +### Installation + +The recommended way is a one-line pipe — **no clone, no download, no compiler**. +The installer pulls prebuilt binaries straight from GitHub Releases. + +**Linux & macOS:** + +```bash +curl -fsSL https://raw.githubusercontent.com/catalystctl/catcode/master/install.sh | bash +``` + +Including the web frontend: ```bash -bash install.sh # download + install catcode and catcode-core -bash install.sh --with-web # …also download + install the web service -bash install.sh --version 0.2.0 # pin a specific release -bash install.sh --dry-run # preview the full plan, execute nothing +curl -fsSL https://raw.githubusercontent.com/catalystctl/catcode/master/install.sh | bash -s -- --with-web ``` -Then run `catcode` from any directory. The workspace is your current directory — -launch it from another folder to work on a different project. +Then run `catcode` from any folder to work on that project. Other options: + +```bash +... | bash -s -- --version 0.2.0 # pin a release +... | bash -s -- --dry-run # preview the plan, execute nothing +... | bash -s -- --uninstall # remove everything +```
install.sh options @@ -112,88 +157,94 @@ launch it from another folder to work on a different project. | `--prefix ` | `/usr/local/bin` | Binary install directory | | `--port ` | `49283` | Web service port | | `--host ` | `0.0.0.0` | Web bind host | -| `--repo ` | — | (source path) Clone `` first, then install from it | +| `--repo ` | — | Clone `` first, then install from it | | `--update` | — | Re-download latest + reinstall (+ restart the service) | | `--uninstall` | — | Stop + remove binaries, service, and state | | `--dry-run` | off | Print the plan, execute nothing | -| `-h`, `--help` | — | Show help |
-| | Linux | macOS | -|:---|:---|:---| -| **TUI / core** | `catcode`, `catcode-core` → `$PREFIX` (sudo) | same | -| **Web service** | systemd unit `catalyst-code-web.service` — starts at boot, auto-restarts | launchd agent `~/Library/LaunchAgents/com.catalyst-code.web.plist` — starts at login, `KeepAlive` | -| **Web logs** | `journalctl -u catalyst-code-web.service -f` | `~/Library/Logs/catalyst-code-web.log` | +**Windows:** -```bash -bash install.sh --update # re-download latest + reinstall (+ restart service) -bash install.sh --uninstall # stop + remove binaries, service, and state +```powershell +irm https://raw.githubusercontent.com/catalystctl/catcode/master/install.ps1 | iex ``` -### Windows — MSI + `install-web.ps1` +Including the web frontend (pass arguments via the scriptblock form, since `iex` +cannot forward parameters): -Windows has no POSIX `install.sh`; use the PowerShell scripts instead. +```powershell +& ([scriptblock]::Create((irm https://raw.githubusercontent.com/catalystctl/catcode/master/install.ps1))) -WithWeb +``` -**TUI** — install `catcode` + `catcode-core` via one of: +No admin, no compiler. Open a NEW PowerShell window (so PATH reloads) and run +`catcode`. Other options: `-Version`, `-BaseUrl`, `-Port`, `-BindHost`, +`-WebDir`, `-Update`, `-Uninstall`, `-DryRun` (see `install.ps1 -Help`). -- **Per-user MSI** (`catcode--windows.msi`) — double-click, or - `msiexec /i catcode--windows.msi`. Installs to - `%LOCALAPPDATA%\Programs\catcode` and adds it to PATH. No admin, clean - Add/Remove Programs entry, in-place upgrades. -- **Standalone `.exe`** (`catcode--windows-x86_64.exe`) — core embedded, no - install; run from any CWD. -- **No-build fallback** `packaging/windows/install.ps1` — copies two raw `.exe` - files to PATH. +
+Windows alternatives (MSI / standalone .exe) -```powershell -msiexec /i catcode--windows.msi # interactive (no UAC) -msiexec /i catcode--windows.msi /quiet # silent -``` +* **Per-user MSI** (`catcode--windows.msi`) — no admin, clean + Add/Remove Programs entry, in-place upgrades: + ```powershell + msiexec /i catcode--windows.msi # interactive (no UAC) + msiexec /i catcode--windows.msi /quiet # silent + ``` +* **Standalone `.exe`** (`catcode--windows-x86_64.exe`) — core embedded, + no install; run from any directory. -**Web service** — `packaging/windows/install-web.ps1` **downloads a prebuilt -web bundle** (and `catcode-core.exe` if not already present) and installs it as -a **Windows Service via [NSSM](https://nssm.cc)** (starts at boot, auto-restarts, -runs with no user logged in) or, if NSSM isn't installed, a **Scheduled Task at -logon** with a restart-loop wrapper (zero extra deps). No `next build`; needs -Node or Bun to run. Add `-BuildFromSource` to compile from a checkout instead. +
-```powershell -pwsh -ExecutionPolicy Bypass -File packaging\windows\install-web.ps1 -# options: -Port 49283 -BindHost 0.0.0.0 -pwsh -ExecutionPolicy Bypass -File packaging\windows\install-web.ps1 -Uninstall -``` +> **Private repo?** The one-liners above fetch the installers from +> `raw.githubusercontent.com`, which works once the repo is **public**. If it +> is private, clone the repo and run `bash install.sh` / `pwsh -File install.ps1` +> locally, or point the installer at a public mirror with `--base-url ` +> (Linux/macOS) / `-BaseUrl ` (Windows). + +
+Prebuilt binaries (standalone, no installer) + +| Platform | Artifact | Run | +|:---|:---|:---| +| **Linux** | `catcode--.AppImage` | `./catcode--x86_64.AppImage` | +| **macOS** | `catcode--macos-{arm64,x86_64}.dmg` | mount → "Install catcode.command" | +| **Windows** | `catcode--windows.msi` | double-click / `msiexec` | + +Each platform also ships a **standalone executable** with the Rust core embedded +(`-tags embed_core`) — one file, no install, run from any directory. + +
+ +

(back to top)

-Logs: `%LOCALAPPDATA%\catalyst-code\catalyst-code-web.log`. +## Usage -### First run +### Terminal — first run ```bash -catcode # launches in the current directory (your workspace) +catcode # launches in the current directory (your workspace) ``` In the TUI: -- `/login` — pick a provider (Umans / OpenAI / Gemini / Anthropic). An API key in - an env var logs in instantly; otherwise it prompts. Subscription accounts use - OAuth (no key) — see [Providers and login](#providers-and-login). -- `/model [N|substr]` — list models, or switch (`/model 3`, `/model glm-5.2`). -- type a prompt to chat. -- `/help` — all commands. +* `/login` — pick a provider. An API key in an env var logs in instantly; + subscription accounts use OAuth (no key). +* `/model [N|substr]` — list models, or switch (`/model 3`, `/model glm-5.2`). +* `/approval never|destructive|always` — change the safety gate. +* type a prompt to chat. `/help` lists every command. -### Web frontend (as a service) +### Web frontend -The Next.js web frontend is the browser equivalent of the TUI — it spawns one +The Next.js web app is the browser equivalent of the TUI — it spawns one `catcode-core` and streams events to the browser over SSE. The installer -downloads a **prebuilt** standalone bundle (no `next build` on the host); it -only needs a Node or Bun runtime to run. `catcode-core` is installed -alongside it (or auto-downloaded on Windows). +downloads a **prebuilt** standalone bundle (no `next build` on the host); it only +needs a Node or Bun runtime to run. | Platform | Install command | Service manager | |:---|:---|:---| -| **Linux** | `bash install.sh --with-web` | systemd (boot-start) | -| **macOS** | `bash install.sh --with-web` | launchd (login-start) | -| **Windows** | `packaging/windows/install-web.ps1` | NSSM service, or scheduled task | +| **Linux** | `curl -fsSL .../install.sh \| bash -s -- --with-web` | systemd (boot-start) | +| **macOS** | `curl -fsSL .../install.sh \| bash -s -- --with-web` | launchd (login-start) | +| **Windows** | `& ([scriptblock]::Create((irm .../install.ps1))) -WithWeb` | NSSM service, or scheduled task | **Manual run (any platform, no service wrapper):** @@ -209,78 +260,19 @@ Set `CATCODE_CORE=` if the core isn't found automatically (it searches > For public exposure, bind to `127.0.0.1` and put a TLS reverse proxy > (Caddy / nginx / IIS) in front. -### Prebuilt binaries (standalone) +

(back to top)

-Prefer a single file with no installer? Grab a prebuilt artifact (built by the -`release-*.sh` scripts; see [Releases](#releases)): +## Providers and Login -| Platform | Artifact | Run | -|:---|:---|:---| -| **Linux** | `catcode--.AppImage` | `./catcode--x86_64.AppImage` | -| **macOS** | `catcode--macos-{arm64,x86_64}.dmg` | mount → "Install catcode.command" | -| **Windows** | `catcode--windows.msi` | double-click / `msiexec` | - -Each platform also ships a **standalone executable** with the Rust core embedded -(`-tags embed_core`) — one file, no install, run from any CWD. - ---- - -## Features - -### Safety - -- **Workspace confinement** — every file op resolves against a workspace root; - absolute paths, `..`, and symlink escapes are rejected. `bash` runs with - `cwd = workspace`. -- **Human-in-the-loop approval** — destructive tools (`bash`, `write_file`, - `edit`, …) require consent under the default `destructive` mode: - `y` approve once · `a` approve and stop asking for this kind · `n` deny. - Modes: `never` / `destructive` / `always`, switchable via `/approval`. - Restricted paths (`.env`, `.git`, `.ssh`, …) are approval-gated for reads - *and* writes. -- **Optional hard sandbox** (Linux) — `--sandbox firejail` wraps bash in a - firejail profile (workspace + shell paths only, dropped caps/seccomp); - `--no-network` adds `unshare -n`. The denylist is a tripwire on top. - -### Robustness - -- **HTTP retry/backoff** — 429, 5xx, and transport errors retried with - exponential backoff (0.5s→8s), honoring `Retry-After`. -- **Idle stream timeout** — `--idle-timeout` (default 120s); a stuck stream - aborts instead of hanging. -- **Context compaction** — at 70% of the model window, oldest tool results are - dropped (system + recent turns kept) with **orphaned-tool-call sanitization** - so a compacted history never sends `tool_calls` without matching results. -- **File-size guards** — `read_file` refuses >5 MiB / 10k lines (with - `offset`/`limit` pagination); `grep`/`glob` cap results. -- **Crash-safe sessions** — append-only JSONL, fsync per message, atomic - rewrites; core-crash auto-recovery on restart. - -### Tooling - -Search-and-replace `edit` (exact, unique, atomic, multi-op) · `grep` + `glob` · -async `bash` (timeout, kill, denylist, 32 KB output cap) · `patch` · -`diagnostics` (cargo check / tsc / go build / py_compile) · `fetch` (read-only -HTTP, egress-controlled) · `todo_write`/`todo_read` · `memory` · git tools · -`spawn` · `subagent` delegation. - -### Observability and persistence - -JSONL debug log (`--debug-log`) · per-turn metrics (TTFT, elapsed, tokens -in/out, TPS) · per-workspace sessions under -`~/.config/catalyst-code/sessions//` with `/sessions` · `/new` · -`/undo` · `/compact` · `/stats`. - ---- - -## Providers and login - -`/login` opens a picker of the bundled presets. You can be logged into several at -once; `/models` lists every provider's models (tagged `[umans]`, `[openai]`, +`/login` opens a picker of the bundled presets. You can be logged into several +at once; `/models` lists every provider's models (tagged `[umans]`, `[openai]`, `[gemini]`, `[anthropic]`), and any model you pick routes that turn to its endpoint. -| Preset | Kind | Endpoint | Key env var | +
+Provider presets + +| Preset | Wire | Endpoint | Key env var | |:---|:---|:---|:---| | **Umans (GLM-5.2)** | OpenAI | `api.code.umans.ai/v1` | `UMANS_API_KEY` | | **OpenAI (Codex)** | OpenAI | `api.openai.com/v1` | `OPENAI_API_KEY` | @@ -290,70 +282,70 @@ endpoint. Keys are persisted per-provider (the env-var *name* is stored when a key came from the environment, so the secret never lands in a config file). +
+ ### Subscription login (OAuth) — no API key -ChatGPT Plus/Pro (Codex), Google One AI (Gemini), and Claude Pro/Max are accessed +ChatGPT Plus/Pro (Codex), Google One AI (Gemini), and Claude Pro/Max are reached via **OAuth**, performed by `/login` itself (no official CLI needed): -- **Gemini** — authorization-code + PKCE + loopback-redirect (opens - accounts.google.com). Reuses `gcloud auth application-default login` if present. -- **Anthropic Claude** — authorize + PKCE + loopback-redirect (opens claude.ai). - Reuses the `claude` CLI token if present. -- **OpenAI Codex** — ⚠️ not yet wired (the ChatGPT token needs the Responses API, - a different request shape). Codex stays on `OPENAI_API_KEY` for now. +* **Gemini** — authorization-code + PKCE + loopback redirect. Reuses + `gcloud auth application-default login` if present. +* **Anthropic Claude** — authorize + PKCE + loopback redirect. Reuses the + `claude` CLI token if present. +* **OpenAI Codex** — ⚠️ not yet wired (the ChatGPT token needs the Responses + API, a different request shape). Codex stays on `OPENAI_API_KEY` for now. Tokens are stored at `~/.config/catalyst-code/oauth/.json` (`0600`) and refreshed automatically. An explicit API key always takes precedence over OAuth. ---- +

(back to top)

-## Build from source +## Architecture -For development (no install, run from the repo): +Four cooperating components around one stdio JSONL protocol: -```bash -cd core && cargo build --release # -> core/target/release/core -cd tui && go build -o catcode # -> tui/catcode -./tui/catcode # the TUI finds ../core/target/release/core -``` - -Requires Rust (stable) and **Go 1.24.2+**. The web frontend needs Bun or Node.js -(see [Web frontend](#web-frontend-as-a-service)). +| Component | Language | Role | +|:---|:---|:---| +| **`core/`** | Rust (async, tokio) | The engine — conversation, model streaming, the agentic tool loop with an approval gate, sessions, memory, plugins, and subagents. | +| **`tui/`** | Go · [Bubble Tea](https://github.com/charmbracelet/bubbletea) | The terminal interface (`catcode`). Spawns the core, streams events, renders approvals and metrics. | +| **`sdk/`** | TypeScript | A thin pi-compatible wrapper (`@catalyst-code/coding-agent`) so the web frontend can drive the core. | +| **`web/`** | Next.js 15 · React 19 | The browser equivalent of the TUI — an SSE bridge to one core process. See [`web/README.md`](web/README.md). | ---- +``` +core/ Rust async engine (stdio JSONL) tui/ Go + Bubble Tea terminal UI +sdk/ TypeScript pi-compatible wrapper web/ Next.js web frontend (SSE bridge) +packaging/ per-platform install scripts .catalyst-code/ bundled agents, plugins, skills +``` -## Architecture +
+core/ source layout ``` -core/ Rust async engine (stdio JSONL) - src/main.rs entry, State, turn loop, approval gate, compaction, ask - src/provider.rs OpenAI/Anthropic streaming, retry/backoff, model discovery, sanitize - src/subagent.rs subagent execution (single/parallel/chain), forked context, depth cap - src/intercom.rs peer intercom bus (contact_supervisor / intercom ask/receive/reply) - src/plugins.rs plugin manager + hooks (pre_*/post_*/lifecycle/pre_turn) - src/protocol.rs wire types (Command / Event) + line emit - src/config.rs CLI + env + JSON config, approval modes, providers - src/workspace.rs path confinement (absolute/.. /symlink rejection) - src/tools.rs tool schemas + classification + execution - src/session.rs append-only JSONL session persistence - src/memory.rs persistent memory store (injected into the system prompt) - src/git_ctx.rs git status/branch context for the system prompt - src/vision.rs vision model config + image attachment - src/fetch_tool.rs HTTP fetch tool (read-only, egress-controlled) - src/oauth.rs OAuth flows (Gemini, Anthropic, Codex) - src/logging.rs JSONL debug log + token estimation - src/staging.rs global default-file staging (~/.catalyst-code/) -tui/ Go + Bubble Tea terminal UI (spawns the core) -sdk/ TypeScript pi-compatible SDK wrapper -web/ Next.js web frontend (SSE bridge) — see web/README.md -packaging/ per-platform install scripts + packaging (linux/ macos/ windows/) -.catalyst-code/ bundled agents, plugins, skills (shipped defaults) -.github/workflows/ CI (core clippy/test, tui vet/test/build + cross-compile, docker) +src/main.rs entry, State, turn loop, approval gate, compaction, ask +src/provider.rs OpenAI/Anthropic streaming, retry/backoff, model discovery, sanitize +src/subagent.rs subagent execution (single/parallel/chain), forked context, depth cap +src/intercom.rs peer intercom bus (contact_supervisor / intercom ask/receive/reply) +src/plugins.rs plugin manager + hooks (pre_*/post_*/lifecycle/pre_turn) +src/protocol.rs wire types (Command / Event) + line emit +src/config.rs CLI + env + JSON config, approval modes, providers +src/workspace.rs path confinement (absolute/.. /symlink rejection) +src/tools.rs tool schemas + classification + execution +src/session.rs append-only JSONL session persistence +src/memory.rs persistent memory store (injected into the system prompt) +src/git_ctx.rs git status/branch context for the system prompt +src/vision.rs vision model config + image attachment +src/fetch_tool.rs HTTP fetch tool (read-only, egress-controlled) +src/oauth.rs OAuth flows (Gemini, Anthropic, Codex) +src/logging.rs JSONL debug log + token estimation +src/staging.rs global default-file staging (~/.catalyst-code/) ``` ---- +
+ +

(back to top)

-## Subagents and intercom +## Subagents and Intercom A port of [`pi-subagents`](https://github.com/nicobailon/pi-subagents) is built into the core. The orchestrator delegates to focused child agents via the @@ -364,60 +356,84 @@ peers over an in-process intercom bus. `researcher` · `planner` · `worker` · `reviewer` · `context-builder` · `oracle` · `delegate`. -**Execution modes:** single `{ agent, task }` · parallel `{ tasks, concurrency }` -· chain `{ chain: [...] }` (with `{previous}`/`{outputs.name}` templating), plus -management actions (`list`/`get`/`create`/`update`/`delete`/`status`/`interrupt`/ -`resume`/`peek`/`steer`/`doctor`). +**Execution modes:** + +```ts +{ agent: "worker", task: "refactor auth" } // single +{ tasks: [{ agent: "scout", task: "a" }, { ... }], concurrency: 2 } // parallel +{ chain: [{ agent: "scout" }, { agent: "planner" }, { agent: "worker" }] } // chain +// management: list / get / create / status / interrupt / resume / peek / steer +``` **Intercom:** -- `contact_supervisor({ reason: "need_decision", message })` — a subagent asks +* `contact_supervisor({ reason: "need_decision", message })` — a subagent asks the orchestrator a blocking question (surfaces as a TUI prompt). -- `intercom({ action: "send"|"ask"|"receive"|"reply"|"targets", to, message })` +* `intercom({ action: "send"|"ask"|"receive"|"reply"|"targets", to, message })` — peer-to-peer messaging between parallel subagents. -**Slash commands:** `/run` · `/parallel` · `/chain` · `/subagents` · -`/subagents-status`. Config lives under `subagents` in settings JSON -(`maxSubagentDepth`, `intercomBridge.mode`, `parallel.maxTasks`, …). +

(back to top)

---- +## Roadmap -## Releases +- [x] Multi-provider login (Umans, OpenAI, Gemini, Anthropic) +- [x] Subagents + intercom bus +- [x] Plugin system (hooks + custom tools, no MCP) +- [ ] Wire OpenAI Codex subscription OAuth (Responses API) +- [ ] More provider presets (local gateways, additional OAuth flows) +- [ ] macOS/Windows sandboxing options +- [ ] Broader plugin / skill ecosystem -`release-all.sh [version]` builds **all** distributable artifacts at once — -Windows MSI + standalone `.exe`, macOS standalone + `.dmg` (arm64 + x86_64), -Linux standalone + AppImage — running each platform script independently and -reporting per-platform pass/fail (a host with a partial toolchain still ships -what it can). `release-web.sh` builds the cross-platform prebuilt web bundle. -All artifacts are published to a GitHub Release by `.github/workflows/release.yml` -(on a `v*` tag push); the installers download them so users never compile. +See the [open issues](https://github.com/catalystctl/catcode/issues) for a full +list of proposed features (and known issues). -| Script | Outputs | -|:---|:---| -| `release-linux.sh` | standalone `catcode--linux-` + `.AppImage` + `catcode-core--linux-` | -| `release-macos.sh` | standalone `catcode--macos-{arm64,x86_64}` + `.dmg` + `catcode-core--macos-{arm64,x86_64}` (cross-compiles via `cargo zigbuild`) | -| `release-windows.sh` | `catcode--windows.msi` + standalone `.exe` + `.zip` + `catcode-core--windows-x86_64.exe` (cross-compiles `x86_64-pc-windows-gnu`) | -| `release-web.sh` | `catcode-web-.tar.gz` — prebuilt Next.js standalone bundle (one cross-platform tarball) | +

(back to top)

+ +## Contributing + +Contributions are welcome. This is a young project — issues and PRs that improve +safety, provider coverage, or docs are especially useful. + +1. Fork the Project +2. Create your Feature Branch (`git checkout -b feature/AmazingFeature`) +3. Commit your Changes (`git commit -m 'Add some AmazingFeature'`) +4. Push to the Branch (`git push origin feature/AmazingFeature`) +5. Open a Pull Request -Each standalone embeds the Rust core via `go:embed` (`-tags embed_core`) so it's -one self-contained file. The separate `catcode-core-*` binaries are for the web -service's `CATCODE_CORE`. Artifacts land in `dist/` with `.sha256` checksums. +### Development setup ---- +Run from the repo (no install needed): -## Protocol +```bash +cd core && cargo build --release # -> core/target/release/core +cd tui && go build -o catcode # -> tui/catcode +./tui/catcode # finds ../core/target/release/core +``` + +Requires Rust (stable) and **Go 1.24+**. The web frontend needs Bun or Node.js +(see [Usage → Web frontend](#usage)). + +### Running tests + +```bash +cd core && cargo test --locked # 300+ unit tests (edit, confinement, bash, sanitize, session, …) +cd tui && go test ./... # TUI tests (handlers, blocks, mention, modal, intercom) +``` + +CI (`.github/workflows/ci.yml`) runs core clippy/test, tui vet/test/build, a Go +cross-compile matrix (linux / darwin / windows), and a Docker image build. + +### Wire protocol Core reads commands from stdin and writes events to stdout — one JSON object per -line. +line. This is the integration point for alternative frontends: ```json {"type":"init"} {"type":"login","preset":"openai","api_key":"sk-..."} {"type":"send","prompt":"...","model":"umans-glm-5.2","reasoning_effort":"high"} {"type":"steer","prompt":"..."} -{"type":"abort"} {"type":"approve","request_id":"","decision":"yes|no|always"} -{"type":"set_approval","mode":"never|destructive|always"} ``` **Events:** `ready` · `authed` · `thinking` · `delta` · `tool_call_start` · @@ -425,37 +441,93 @@ line. `tool_result` · `compacted` · `http_retry` · `metrics` · `approval_changed` · `done` · `aborted` · `reset` · `error` (plus subagent / memory / session events). ---- +
+Releases -## Testing +`release-all.sh [version]` builds **all** distributable artifacts at once and +reports per-platform pass/fail (a host with a partial toolchain still ships what +it can). All artifacts are published to a GitHub Release by +`.github/workflows/release.yml` on a `v*` tag push; the installers download them +so users never compile. Each standalone embeds the Rust core via `go:embed` +(`-tags embed_core`); the separate `catcode-core-*` binaries are for the web +service's `CATCODE_CORE`. -```bash -cd core && cargo test --locked # 314 unit tests (edit, confinement, bash, sanitize, session, …) -cd tui && go test ./... # TUI tests (handlers, blocks, mention, modal, intercom) -``` +| Script | Outputs | +|:---|:---| +| `release-linux.sh` | standalone `catcode--linux-` + `.AppImage` + `catcode-core--linux-` | +| `release-macos.sh` | standalone `catcode--macos-{arm64,x86_64}` + `.dmg` + `catcode-core--macos-{arch}` (cross-compiles via `cargo zigbuild`) | +| `release-windows.sh` | `catcode--windows.msi` + standalone `.exe` + `.zip` + `catcode-core--windows-x86_64.exe` (cross-compiles `x86_64-pc-windows-gnu`) | +| `release-web.sh` | `catcode-web-.tar.gz` — prebuilt Next.js standalone bundle (one cross-platform tarball) | -CI (`.github/workflows/ci.yml`) runs core clippy/test, tui vet/test/build, a Go -cross-compile matrix (linux / darwin / windows), and a Docker image build. +Artifacts land in `dist/` with `.sha256` checksums. ---- +
-## Security and notes +
+Security notes -- **OpenAI-compatible** — point `--base-url` at any OpenAI-shaped endpoint. - Umans-specific logic (GLM `reasoning_effort=high` clamp, `reasoning_content` - replay, `/models/info` discovery) is isolated to `provider.rs`. -- **No fixed turn cap** — the ceiling is the session token budget +* **OpenAI-compatible** — point `--base-url` at any OpenAI-shaped endpoint. + Provider-specific logic (GLM `reasoning_effort=high` clamp, + `reasoning_content` replay, `/models/info` discovery) is isolated to + `provider.rs`. +* **No fixed turn cap** — the ceiling is the session token budget (`--max-session-tokens`, `0` = unlimited). The model can call `finish` to exit - cleanly or `spawn` a nested sub-agent. -- **Hard security boundary** — pass `--sandbox firejail --no-network` (or set in - the TUI settings modal). The denylist is a tripwire on top; workspace - confinement covers file paths, but `bash` is only sandboxed when `--sandbox` is - set. Sandboxing is **Linux-only**; on macOS/Windows leave it `none`. -- **Windows bash** — the agent's `bash` tool needs bash on PATH (Git Bash or + cleanly or `spawn` a nested agent. +* **Hard security boundary** — pass `--sandbox firejail --no-network` (or set it + in the TUI settings). The denylist is a tripwire on top; workspace confinement + covers file paths, but `bash` is only sandboxed when `--sandbox` is set. + Sandboxing is **Linux-only**. +* **Windows bash** — the agent's `bash` tool needs bash on PATH (Git Bash or WSL); chat and the file tools work without it. ---- +
+ +

(back to top)

## License -[MIT](LICENSE) © karutoil +Distributed under the MIT License. See [`LICENSE`](LICENSE) for more information. + +

(back to top)

+ +## Contact + +karutoil — [github.com/karutoil](https://github.com/karutoil) + +Project Link: [https://github.com/catalystctl/catcode](https://github.com/catalystctl/catcode) + +

(back to top)

+ +## Acknowledgments + +* [pi-subagents](https://github.com/nicobailon/pi-subagents) — the subagent + + intercom design this project's orchestration is ported from. +* [Bubble Tea](https://github.com/charmbracelet/bubbletea) (TUI), + [Next.js](https://nextjs.org) (web), and the Rust async ecosystem. +* [Best-README-Template](https://github.com/othneildrew/Best-README-Template) — + README structure inspiration. + +

(back to top)

+ + + +[contributors-shield]: https://img.shields.io/github/contributors/catalystctl/catcode.svg?style=for-the-badge +[contributors-url]: https://github.com/catalystctl/catcode/graphs/contributors +[forks-shield]: https://img.shields.io/github/forks/catalystctl/catcode.svg?style=for-the-badge +[forks-url]: https://github.com/catalystctl/catcode/network/members +[stars-shield]: https://img.shields.io/github/stars/catalystctl/catcode.svg?style=for-the-badge +[stars-url]: https://github.com/catalystctl/catcode/stargazers +[issues-shield]: https://img.shields.io/github/issues/catalystctl/catcode.svg?style=for-the-badge +[issues-url]: https://github.com/catalystctl/catcode/issues +[license-shield]: https://img.shields.io/github/license/catalystctl/catcode.svg?style=for-the-badge +[license-url]: https://github.com/catalystctl/catcode/blob/master/LICENSE +[website-shield]: https://img.shields.io/badge/website-code.catalystctl.com-ff9e28?style=for-the-badge +[website-url]: https://code.catalystctl.com +[Rust-badge]: https://img.shields.io/badge/Rust-stable-ce422b?style=for-the-badge&logo=rust&logoColor=white +[Rust-url]: https://www.rust-lang.org/ +[Go-badge]: https://img.shields.io/badge/Go-1.24%2B-00add8?style=for-the-badge&logo=go&logoColor=white +[Go-url]: https://go.dev/ +[Next.js-badge]: https://img.shields.io/badge/Next.js-15-000000?style=for-the-badge&logo=nextdotjs&logoColor=white +[Next.js-url]: https://nextjs.org/ +[TypeScript-badge]: https://img.shields.io/badge/TypeScript-5-3178c6?style=for-the-badge&logo=typescript&logoColor=white +[TypeScript-url]: https://www.typescriptlang.org/ diff --git a/core/Cargo.lock b/core/Cargo.lock index bcf78c1..46c8501 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -61,8 +61,10 @@ name = "catalyst-code-core" version = "0.2.0" dependencies = [ "base64", + "filetime", "futures-util", "jsonwebtoken", + "libc", "rand 0.8.6", "regex", "reqwest", @@ -151,6 +153,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" diff --git a/core/Cargo.toml b/core/Cargo.toml index 6972c80..7f5ee3d 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -21,6 +21,10 @@ sha2 = "0.10" base64 = "0.22" rand = "0.8" jsonwebtoken = "9" +libc = "0.2" + +[dev-dependencies] +filetime = "0.2" [profile.release] opt-level = 2 diff --git a/core/src/config.rs b/core/src/config.rs index 33dc96f..65ed3b3 100644 --- a/core/src/config.rs +++ b/core/src/config.rs @@ -127,16 +127,22 @@ pub struct Config { pub max_read_lines: usize, pub context_compact_at: f32, // fraction of context_window that triggers compaction pub context_digest_at: f32, // fraction of context_window that triggers stale-tool-result digesting (sub-threshold reclaim; 0 disables) + pub auto_compact: bool, // automatically compact when context approaches the limit (threshold + idle); manual /compact always works regardless + /// Opt-in JSONL debug log path. Records every tool call with its full + /// arguments (file contents, bash commands) — which may include secrets the + /// model writes (e.g. into a `.env`). User-owned, off by default, rotates at + /// 64 MiB. Enable only when debugging. pub debug_log: Option, pub session_file: Option, pub default_model: Option, // --- production knobs (items 3,4,7) --- - pub sandbox: Sandbox, // --sandbox firejail wraps bash - pub no_network: bool, // --no-network: unshare -n on bash - pub idle_timeout_secs: u64, // per-chunk SSE idle timeout - pub max_session_tokens: u64, // hard session token budget (0 = unlimited) - pub summarize_on_compact: bool, // use a model call to summarize dropped turns - pub rolling_state: bool, // inject a transient tail work-state summary (KV-cache-aware) + pub sandbox: Sandbox, // --sandbox firejail wraps bash + pub no_network: bool, // --no-network: unshare -n on bash + pub idle_timeout_secs: u64, // per-chunk SSE idle timeout + pub max_session_tokens: u64, // hard session token budget (0 = unlimited) + pub summarize_on_compact: bool, // use a model call to summarize dropped turns + pub compact_instructions: Option, // optional guidance woven into the summarize prompt (e.g. "Focus on code samples and API usage"); /compact overrides per-call + pub rolling_state: bool, // inject a transient tail work-state summary (KV-cache-aware) /// Auto-reflect: on a non-trivial turn (≥ `auto_reflect_min_tool_calls` tool /// calls), inject a reflection continuation before `finish` exits so durable /// facts get persisted (memory) and recurring patterns get written as skills @@ -559,7 +565,16 @@ pub fn save_providers_config( .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::NotFound, "no home directory"))?; if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700)); + } } + // Cross-process lock: this is a read-modify-write (merge providers into + // existing config.json). Two processes logging in different providers + // concurrently would otherwise race and one provider's entry would be lost. + let _lock = crate::fsutil::FileLock::acquire(&path.with_extension("lock"))?; // Read existing config.json (if any) and merge so other keys survive. let mut root: Value = std::fs::read_to_string(&path) .ok() @@ -574,9 +589,8 @@ pub fn save_providers_config( root["activeProvider"] = json!(a); } let data = serde_json::to_string_pretty(&root).unwrap_or_default(); - let tmp = path.with_extension("json.tmp"); - std::fs::write(&tmp, data)?; - std::fs::rename(&tmp, &path)?; + // Unique-temp atomic write + 0600 (fsutil): no shared-temp collision. + crate::fsutil::atomic_write_secure(&path, data.as_bytes())?; Ok(()) } @@ -725,7 +739,7 @@ impl Default for Config { ], max_read_bytes: 5_242_880, // 5 MiB (was 1 MiB; real files exceed 1MB) max_read_lines: 10_000, // was 2000; pagination covers the rest - context_compact_at: 0.70, + context_compact_at: 0.90, context_digest_at: 0.40, debug_log: None, session_file: None, @@ -735,6 +749,8 @@ impl Default for Config { idle_timeout_secs: 120, // some reasoning models think >60s before first token max_session_tokens: 0, summarize_on_compact: true, + compact_instructions: None, + auto_compact: true, rolling_state: true, auto_reflect: true, auto_reflect_min_tool_calls: 1, @@ -1017,6 +1033,28 @@ pub fn load() -> Config { c.auto_reflect_min_tool_calls = n.max(1); } } + // auto_compact: toggle automatic context compaction (threshold-triggered + + // idle). Default true. Manual /compact always works regardless of this + // setting. Mirrors Claude Code's autoCompactEnabled / DISABLE_AUTO_COMPACT. + if let Ok(v) = std::env::var("CATALYST_CODE_AUTO_COMPACT") { + let on = v.is_empty() || v == "1" || v.eq_ignore_ascii_case("true"); + let off = v == "0" || v.eq_ignore_ascii_case("false"); + if off { + c.auto_compact = false; + } else if on { + c.auto_compact = true; + } + } + // compact_instructions: optional guidance woven into the compaction summarize + // prompt ("Focus on code samples and API usage"). /compact + // overrides per-call; this sets the default used by auto-compaction. + if let Ok(v) = std::env::var("CATALYST_CODE_COMPACT_INSTRUCTIONS") { + if v.trim().is_empty() { + c.compact_instructions = None; + } else { + c.compact_instructions = Some(v); + } + } // Custom providers. `UMANS_PROVIDERS` is a JSON array of provider objects // (same shape as the config-file `providers` field); merged after the file @@ -1124,6 +1162,19 @@ fn apply_json(c: &mut Config, v: &Value) { if let Some(b) = v.get("summarize_on_compact").and_then(|x| x.as_bool()) { c.summarize_on_compact = b; } + if let Some(b) = v.get("auto_compact").and_then(|x| x.as_bool()) { + c.auto_compact = b; + } + if let Some(s) = v.get("compact_instructions").and_then(|x| x.as_str()) { + c.compact_instructions = if s.trim().is_empty() { + None + } else { + Some(s.to_string()) + }; + } + if let Some(f) = v.get("context_compact_at").and_then(|x| x.as_f64()) { + c.context_compact_at = f as f32; + } if let Some(b) = v.get("rolling_state").and_then(|x| x.as_bool()) { c.rolling_state = b; } diff --git a/core/src/fetch_tool.rs b/core/src/fetch_tool.rs index d32efc5..9ecbe15 100644 --- a/core/src/fetch_tool.rs +++ b/core/src/fetch_tool.rs @@ -2,10 +2,12 @@ // NOT subject to the bash sandbox / `--no-network` (`unshare -n` only wraps the // bash command), so the agent can still look up docs under the hard-security // config. A host allowlist (`cfg.fetch_allowlist`) restricts egress; empty -// allowlist = any http(s) host. +// allowlist = any public http(s) host (private/loopback/link-local ranges are +// blocked by default as SSRF hardening; an explicit allowlist entry overrides). use crate::config::Config; use crate::tools::{smart_truncate, Outcome}; use serde_json::Value; +use std::net::IpAddr; /// Parse an absolute http(s) URL enough to validate it and extract the host /// (lowercased) for allowlist matching. Returns (scheme, host). No `url` crate @@ -16,8 +18,14 @@ fn parse_http_host(url: &str) -> Option<(String, String)> { if scheme != "http" && scheme != "https" { return None; } - let end = rest.find(&['/', '?', '#', ':'][..]).unwrap_or(rest.len()); - let host = rest[..end].to_ascii_lowercase(); + // IPv6 literal: [::1]:8080 — host is the bracketed content. + let host = if let Some(rest) = rest.strip_prefix('[') { + let end = rest.find(']')?; + rest[..end].to_ascii_lowercase() + } else { + let end = rest.find(&['/', '?', '#', ':'][..]).unwrap_or(rest.len()); + rest[..end].to_ascii_lowercase() + }; if host.is_empty() { return None; } @@ -37,11 +45,54 @@ fn host_matches(host: &str, pattern: &str) -> bool { } } +/// Is `ip` in a private/loopback/link-local/unspecified range? These are +/// blocked by default (empty fetch_allowlist) to harden against SSRF — e.g. +/// cloud-metadata at 169.254.169.254, localhost services, RFC-1918 internals. +/// An explicit fetch_allowlist entry overrides this (operator opt-in). +fn ip_is_private(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => { + let o = v4.octets(); + o[0] == 127 // loopback 127.0.0.0/8 + || (o[0] == 169 && o[1] == 254) // link-local 169.254.0.0/16 (cloud-metadata) + || o[0] == 10 // private 10.0.0.0/8 + || (o[0] == 172 && (16..=31).contains(&o[1])) // private 172.16.0.0/12 + || (o[0] == 192 && o[1] == 168) // private 192.168.0.0/16 + || o[0] == 0 // 0.0.0.0/8 (unspecified + "this network") + } + IpAddr::V6(v6) => { + v6.is_loopback() // ::1 + || v6.is_unspecified() // :: + || (v6.segments()[0] & 0xffc0) == 0xfe80 // link-local fe80::/10 + || (v6.segments()[0] & 0xfe00) == 0xfc00 // unique-local fc00::/7 + || v6 + .to_ipv4() + .map(|v4| ip_is_private(IpAddr::V4(v4))) + .unwrap_or(false) // ::ffff:a.b.c.d (IPv4-mapped) + } + } +} + +/// Is `host` a private address? Checks IP literals directly (no DNS — fast and +/// side-effect-free, so it's safe in the sync redirect policy). A hostname +/// that resolves to a private IP is a residual risk controlled by the +/// allowlist; we deliberately don't do DNS here to keep the check hang-proof. +fn host_is_private(host: &str) -> bool { + match host.parse::() { + Ok(ip) => ip_is_private(ip), + Err(_) => false, + } +} + +/// Decide whether `host` is permitted. A non-empty allowlist means the operator +/// explicitly opted in to exactly those hosts (a listed private host is allowed +/// — explicit opt-in wins). An empty allowlist allows any PUBLIC host; +/// private/loopback/link-local ranges are blocked by default (SSRF hardening). fn host_allowed(host: &str, allowlist: &[String]) -> bool { - if allowlist.is_empty() { - return true; + if !allowlist.is_empty() { + return allowlist.iter().any(|p| host_matches(host, p)); } - allowlist.iter().any(|p| host_matches(host, p)) + !host_is_private(host) } /// A redirect policy that re-checks the fetch_allowlist on EVERY redirect hop, @@ -51,8 +102,9 @@ fn host_allowed(host: &str, allowlist: &[String]) -> bool { /// security model (the whole point: bash stays offline, fetch reaches only /// listed hosts). A redirect whose target host isn't allowed is stopped — the /// 3xx response is returned without following, and the disallowed host is -/// never contacted. Shared by `fetch` and `web_search` so both honor the same -/// redirect policy. +/// never contacted. With an empty allowlist, private/loopback/link-local +/// redirect targets are also stopped (SSRF hardening — see `host_allowed`). +/// Shared by `fetch` and `web_search` so both honor the same redirect policy. pub(crate) fn allowlist_redirect_policy(allowlist: Vec) -> reqwest::redirect::Policy { reqwest::redirect::Policy::custom(move |attempt| { let host = attempt.url().host_str().unwrap_or("").to_ascii_lowercase(); @@ -154,8 +206,13 @@ pub(crate) fn egress_check(label: &str, url: &str, cfg: &Config) -> Option Outcome { ); } if !host_allowed(&host, &cfg.fetch_allowlist) { + if cfg.fetch_allowlist.is_empty() { + return Outcome::err(format!( + "fetch: host '{host}' is a private/loopback/link-local address and is blocked by default (empty fetch_allowlist); add it to fetch_allowlist to explicitly opt in" + )); + } return Outcome::err(format!( - "fetch: host '{host}' is not in the allowlist ({} pattern(s) configured); add it to fetch_allowlist to permit it, or leave the allowlist empty to allow any host", + "fetch: host '{host}' is not in the allowlist ({} pattern(s) configured); add it to fetch_allowlist to permit it", cfg.fetch_allowlist.len() )); } @@ -298,6 +360,10 @@ mod tests { ); assert_eq!(parse_http_host("file:///etc/passwd"), None); assert_eq!(parse_http_host("not a url"), None); + assert_eq!( + parse_http_host("http://[::1]:8080/x"), + Some(("http".into(), "::1".into())) + ); } #[test] @@ -307,7 +373,7 @@ mod tests { assert!(!host_matches("evilrust-lang.org", "*.rust-lang.org")); assert!(host_matches("docs.rs", "docs.rs")); assert!(!host_matches("docs.rs", "crates.io")); - // empty allowlist = allow all + // empty allowlist = allow any public host (private ranges blocked) assert!(host_allowed("anything.example", &[])); let list = vec!["*.rust-lang.org".into(), "docs.rs".into()]; assert!(host_allowed("doc.rust-lang.org", &list)); @@ -315,6 +381,38 @@ mod tests { assert!(!host_allowed("evil.com", &list)); } + #[test] + fn private_ranges_blocked_by_default() { + // empty allowlist: private/loopback/link-local blocked; public allowed. + assert!(!host_allowed("169.254.169.254", &[])); // cloud-metadata + assert!(!host_allowed("127.0.0.1", &[])); // loopback + assert!(!host_allowed("127.255.255.255", &[])); // loopback edge + assert!(!host_allowed("10.0.0.5", &[])); // private 10/8 + assert!(!host_allowed("192.168.1.1", &[])); // private 192.168/16 + assert!(!host_allowed("172.16.0.1", &[])); // private 172.16/12 start + assert!(!host_allowed("172.31.255.255", &[])); // private 172.16/12 end + assert!(host_allowed("172.32.0.1", &[])); // just outside → public + assert!(host_allowed("8.8.8.8", &[])); // public + assert!(host_allowed("1.1.1.1", &[])); // public + assert!(host_allowed("example.com", &[])); // hostname → allowed (no DNS) + // IPv6 + assert!(!host_allowed("::1", &[])); // v6 loopback + assert!(!host_allowed("fe80::1", &[])); // v6 link-local + assert!(!host_allowed("fc00::1", &[])); // v6 unique-local + assert!(!host_allowed("fd00::1", &[])); // v6 unique-local + assert!(!host_allowed("::ffff:169.254.169.254", &[])); // v4-mapped metadata + assert!(host_allowed("2606:4700:4700::1111", &[])); // public v6 + } + + #[test] + fn explicit_allowlist_overrides_private_block() { + // operator explicitly allowlists a private host → allowed (opt-in wins). + let list = vec!["127.0.0.1".into(), "localhost".into()]; + assert!(host_allowed("127.0.0.1", &list)); + assert!(host_allowed("localhost", &list)); + assert!(!host_allowed("169.254.169.254", &list)); // not listed → denied + } + #[test] fn html_strips_tags_and_scripts() { let html = "

Title

Hi & bye <3

"; @@ -385,7 +483,8 @@ mod tests { ); let (url, _h) = mock_http(html, "text/html; charset=utf-8").await; let cfg = crate::config::Config { - fetch_allowlist: Vec::new(), + // 127.0.0.1 is loopback → blocked by default; allowlist the mock host. + fetch_allowlist: vec!["127.0.0.1".into(), "localhost".into()], fetch_timeout_secs: 10, fetch_max_bytes: 1 << 20, ..crate::config::Config::default() diff --git a/core/src/fsutil.rs b/core/src/fsutil.rs new file mode 100644 index 0000000..7824b75 --- /dev/null +++ b/core/src/fsutil.rs @@ -0,0 +1,268 @@ +//! Cross-process-safe filesystem helpers: unique-temp atomic writes and an +//! advisory cross-process file lock. +//! +//! ## Why this exists +//! +//! The harness can run as multiple concurrent processes (two TUI sessions, a +//! TUI + a web server, parallel CI). Several files under +//! `~/.config/catalyst-code/` are SHARED across processes — the models cache, +//! the memory store, the pattern log, OAuth tokens, config.json, settings.json. +//! Two hazards arise when two processes touch the same shared file: +//! +//! 1. **Temp-file collision (corruption).** The atomic-write pattern writes a +//! sibling temp file then renames it over the target. If the temp name is +//! FIXED (e.g. `foo.json.tmp`), two concurrent writers open the SAME temp +//! file, interleave their writes, and one renames a corrupted file over the +//! target. Fix: every temp file gets a unique name (pid + random suffix). +//! +//! 2. **Lost update (read-modify-write).** Several writers read the existing +//! file, merge their change, and write it back. Two concurrent writers both +//! read the same base; the second to rename clobbers the first's change. +//! For accumulating stores (memory) this is silent durable data loss. Fix: +//! a cross-process advisory lock around the read-modify-write critical +//! section. +//! +//! `presence.rs` already solved #1 for its own files (per-pid names). This +//! module generalizes the fix to every shared-file writer. + +use std::io; +use std::path::{Path, PathBuf}; + +/// Build a unique temp-file path beside `target` (same directory). The name is +/// `....tmp` — hidden, unique per process AND per call, so +/// two concurrent writers never share a temp file. A crash mid-write leaves an +/// orphaned hidden temp (benign: small, rare, never read by anyone). +pub fn unique_tmp(target: &Path) -> PathBuf { + use rand::Rng; + let pid = std::process::id(); + let rand: u64 = rand::thread_rng().gen(); + let name = target.file_name().and_then(|n| n.to_str()).unwrap_or("tmp"); + target.with_file_name(format!(".{name}.{pid}.{rand:016x}.tmp")) +} + +/// Atomically write `content` to `target`: a unique temp file is written, +/// fsync'd, then renamed over the target. A crash mid-write leaves the orphaned +/// temp (never a truncated target). The unique temp means concurrent writers +/// never collide on the temp file. +pub fn atomic_write(target: &Path, content: &[u8]) -> io::Result<()> { + use std::io::Write; + let tmp = unique_tmp(target); + { + let mut f = std::fs::File::create(&tmp)?; + f.write_all(content)?; + f.flush()?; + f.sync_all()?; + } + if let Err(e) = std::fs::rename(&tmp, target) { + let _ = std::fs::remove_file(&tmp); + return Err(e); + } + Ok(()) +} + +/// Convenience wrapper for `&str` content. +pub fn atomic_write_str(target: &Path, content: &str) -> io::Result<()> { + atomic_write(target, content.as_bytes()) +} + +/// Like [`atomic_write`] but sets 0600 perms on the file (secrets: OAuth tokens, +/// config with API keys). The temp is chmod'd BEFORE the rename so the target +/// is never briefly world-readable — matching the original oauth/config pattern. +#[cfg(unix)] +pub fn atomic_write_secure(target: &Path, content: &[u8]) -> io::Result<()> { + use std::io::Write; + use std::os::unix::fs::PermissionsExt; + let tmp = unique_tmp(target); + { + let mut f = std::fs::File::create(&tmp)?; + f.write_all(content)?; + f.flush()?; + f.sync_all()?; + std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o600))?; + } + if let Err(e) = std::fs::rename(&tmp, target) { + let _ = std::fs::remove_file(&tmp); + return Err(e); + } + Ok(()) +} + +/// Non-Unix fallback: no permission bits to set. +#[cfg(not(unix))] +pub fn atomic_write_secure(target: &Path, content: &[u8]) -> io::Result<()> { + atomic_write(target, content) +} + +/// An advisory cross-process exclusive lock held for the lifetime of the guard. +/// +/// On Unix this is an `flock(2)` on a sidecar lock file: it blocks until +/// acquired and AUTO-RELEASES when the process exits (even on `kill -9` or a +/// crash), so there are never stale locks. On non-Unix platforms it is a no-op +/// (the unique-temp atomic write still prevents corruption; only the +/// read-modify-write lost-update remains possible, which is acceptable for the +/// rare non-Unix multi-process case). +/// +/// Use it to serialize a read-modify-write critical section on a shared file: +/// +/// ```ignore +/// let _lock = fsutil::FileLock::acquire(&path.with_extension("lock"))?; +/// let existing = std::fs::read_to_string(&path).unwrap_or_default(); +/// // ... merge ... +/// fsutil::atomic_write(&path, &merged)?; +/// // lock released on drop +/// ``` +pub struct FileLock { + // The file handle is held for the lock's lifetime; dropping it closes the + // fd and releases the flock. Never read/written — it exists only to own + // the lock. + #[cfg(unix)] + _file: std::fs::File, + // Mark unused on non-Unix so the field isn't flagged dead code. + #[cfg(not(unix))] + _file: (), +} + +impl FileLock { + /// Block until an exclusive lock on `lock_path` is acquired. Creates the + /// lock file (and its parent dir) if absent. Returns a guard that releases + /// the lock on drop. + pub fn acquire(lock_path: &Path) -> io::Result { + #[cfg(unix)] + { + use std::os::unix::io::AsRawFd; + if let Some(parent) = lock_path.parent() { + let _ = std::fs::create_dir_all(parent); + } + // create(true) so the lock file exists; we never read/write its + // contents — flock only needs an open file description. + let file = std::fs::OpenOptions::new() + .create(true) + .write(true) + .read(true) + // Explicitly non-truncating: this is a sidecar LOCK file whose + // content is never read or written — flock operates on the open + // file description, not the bytes. Truncating would be + // meaningless (and clobber any concurrent holder's file, though + // they don't use the bytes either). + .truncate(false) + .open(lock_path)?; + // Blocking exclusive flock (LOCK_EX). Auto-released on close/exit. + let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) }; + if rc != 0 { + return Err(io::Error::last_os_error()); + } + Ok(FileLock { _file: file }) + } + #[cfg(not(unix))] + { + let _ = lock_path; + Ok(FileLock { _file: () }) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unique_tmp_is_unique_and_beside_target() { + let p = Path::new("/tmp/catalyst_code_fsutil_test.json"); + let a = unique_tmp(p); + let b = unique_tmp(p); + assert_ne!(a, b, "two calls must yield different temp paths"); + assert_eq!(a.parent(), p.parent(), "temp must be in the same dir"); + assert!(a + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("") + .starts_with(".catalyst_code_fsutil_test.json.")); + } + + #[test] + fn atomic_write_roundtrip() { + let dir = + std::env::temp_dir().join(format!("catalyst_code_fsutil_rw_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let p = dir.join("data.json"); + atomic_write(&p, b"hello").unwrap(); + assert_eq!(std::fs::read(&p).unwrap(), b"hello"); + // overwrite + atomic_write(&p, b"world").unwrap(); + assert_eq!(std::fs::read(&p).unwrap(), b"world"); + // no leftover temps + let temps: Vec<_> = std::fs::read_dir(&dir) + .unwrap() + .flatten() + .filter(|e| { + e.file_name() + .to_str() + .map(|n| n.starts_with(".")) + .unwrap_or(false) + }) + .collect(); + assert!(temps.is_empty(), "no orphaned temp files: {temps:?}"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + #[cfg(unix)] + fn atomic_write_secure_sets_0600() { + use std::os::unix::fs::PermissionsExt; + let dir = + std::env::temp_dir().join(format!("catalyst_code_fsutil_sec_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let p = dir.join("secret.json"); + atomic_write_secure(&p, b"{\"key\":\"x\"}").unwrap(); + let mode = std::fs::metadata(&p).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o600, "secure write must be 0600"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + #[cfg(unix)] + fn file_lock_serializes_concurrent_writers() { + // Two threads doing a read-modify-write under the SAME lock must not + // lose updates: the lock serializes them so both increments land. + let dir = + std::env::temp_dir().join(format!("catalyst_code_fsutil_lock_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let data = dir.join("counter.json"); + let lock = dir.join("counter.lock"); + atomic_write(&data, b"0").unwrap(); + + let n = 8usize; + let mut handles = Vec::new(); + for _ in 0..n { + let data = data.clone(); + let lock = lock.clone(); + handles.push(std::thread::spawn(move || { + let _g = FileLock::acquire(&lock).unwrap(); + let cur: u64 = std::fs::read_to_string(&data) + .unwrap() + .trim() + .parse() + .unwrap_or(0); + atomic_write(&data, format!("{}", cur + 1).as_bytes()).unwrap(); + })); + } + for h in handles { + h.join().unwrap(); + } + // Without the lock this would race and lose increments; with it, all 8 + // land and the counter is exactly n. + let final_val: u64 = std::fs::read_to_string(&data) + .unwrap() + .trim() + .parse() + .unwrap(); + assert_eq!( + final_val, n as u64, + "lock must serialize RMW: got {final_val}" + ); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/core/src/main.rs b/core/src/main.rs index f5b393b..17493cc 100644 --- a/core/src/main.rs +++ b/core/src/main.rs @@ -9,6 +9,7 @@ mod config; mod fetch_tool; +mod fsutil; mod git_ctx; mod intercom; mod logging; @@ -17,6 +18,7 @@ mod message; mod oauth; mod pattern_log; mod plugins; +mod presence; mod protocol; mod provider; mod search_tool; @@ -118,6 +120,22 @@ pub fn build_system_prompt(workspace: &std::path::Path, with_skill: bool) -> Str prompt } +/// Build the MAIN agent's system prompt: the base prompt (git context + +/// memory + PLUGIN_DOCS + orchestrator skill manifest) PLUS any text plugins +/// inject via their `system_prompt` manifest field. Plugin injection is empty +/// (so the prompt + its prefix cache are untouched) when no enabled plugin +/// declares one — mirroring how `build_system_prompt` stays cheap in the common +/// case. Subagents do NOT get plugin injection (they use the built-in tool set +/// only), matching the plugin-tools-are-main-agent-scoped design. +fn build_main_system_prompt(workspace: &std::path::Path, pm: &plugins::PluginManager) -> String { + let mut prompt = build_system_prompt(workspace, true); + let inj = pm.system_prompt_injection(); + if !inj.is_empty() { + prompt.push_str(&inj); + } + prompt +} + /// Load the bundled pi-subagents SKILL.md (project then user scope) for the /// orchestrator's system prompt. Returns None if no skill file is found. fn subagent_orchestrator_skill(workspace: &std::path::Path) -> Option { @@ -353,6 +371,14 @@ pub struct State { /// set when `/login` picks the manual flow (SSH/headless) and consumed by /// the `oauth_code` command when the user pastes the code. pub pending_oauth: Mutex>, + /// Cached live peer sessions in this workspace, refreshed every heartbeat + /// (~8s) by the presence task. Kept in-memory so the anomaly nudge in + /// `run_turn` can check for concurrent activity WITHOUT a filesystem read + /// on every tool result (the hot path). Empty when alone. See `presence`. + pub peers: Mutex>, + /// Last time the concurrency anomaly note was emitted, for per-session + /// rate-limiting so a pathological tool-call loop can't nag every result. + pub last_concurrency_note: Mutex>, } /// Shared tail of `login_oauth` (web flow) and `oauth_code` (manual flow): @@ -909,6 +935,90 @@ async fn record_file_touch(st: &State, tool: &str, args: &Value) { emit_work_state(st).await; } +/// Cooldown between concurrency-anomaly notes (seconds). Prevents nagging in a +/// tight tool-call loop; one nudge per minute is enough to change behavior from +/// "fix the phantom error" to "check the neighbors". +const CONCURRENCY_NOTE_COOLDOWN: u64 = 60; + +/// If another session is active in this workspace AND something looks off (a +/// tool failed, or we're touching a file a peer recently touched), surface a +/// short note appended to the tool result — so the agent doesn't assume every +/// error is its own fault and "fix" a neighbor's in-flight work. Uses the cached +/// peer snapshot (refreshed by the heartbeat) so the hot path does NO filesystem +/// read. Rate-limited per session to avoid nagging. +async fn maybe_concurrency_note( + st: &State, + tool_name: &str, + args: &Value, + outcome_ok: bool, +) -> Option { + // Cooldown: at most one note per window. Checked first (before cloning the + // peer snapshot) so a tight loop short-circuits cheaply after the first nudge. + { + let last = st.last_concurrency_note.lock().await; + if let Some(t) = *last { + if t.elapsed() < std::time::Duration::from_secs(CONCURRENCY_NOTE_COOLDOWN) { + return None; + } + } + } + let peers = st.peers.lock().await.clone(); + if peers.is_empty() { + return None; // alone — nothing to surface; don't arm the cooldown + } + let touching = peers_touching(&peers, tool_name, args); + // (a) a tool failed — could be a neighbor leaving the tree inconsistent. + if !outcome_ok { + *st.last_concurrency_note.lock().await = Some(std::time::Instant::now()); + let mut s = format!( + "⚠ {} other agent session(s) are active in this workspace. This error may \ + not be from your changes — another session may have left the tree in an \ + inconsistent state. Consider `workspace_activity` to inspect before \ + 'fixing' it.", + peers.len() + ); + if !touching.is_empty() { + s.push_str(&format!(" Active sessions recently touched: {touching}.")); + } + return Some(s); + } + // (b) a file tool touching a path a peer recently touched (a real conflict). + if !touching.is_empty() { + *st.last_concurrency_note.lock().await = Some(std::time::Instant::now()); + return Some(format!( + "ℹ Another agent session in this workspace recently touched: {touching}. \ + You may be reading/editing in-flight work — consider `workspace_activity` \ + to coordinate." + )); + } + None +} + +/// Return a comma-list of "pid N" for peers whose recent_files contain the +/// current tool's target path. Exact separator-normalized match — precise to +/// avoid false-positive nagging; a miss just means no warning (safe). +fn peers_touching(peers: &[presence::PresenceRecord], tool_name: &str, args: &Value) -> String { + let path = match tool_name { + "read_file" | "edit" | "write_file" | "patch" | "bulk_read" | "bulk_write" + | "bulk_edit" => args.get("path").and_then(|v| v.as_str()).unwrap_or(""), + _ => "", + }; + if path.is_empty() { + return String::new(); + } + let target = path.replace('\\', "/"); + let hitting: Vec<_> = peers + .iter() + .filter(|p| { + p.recent_files + .iter() + .any(|f| f.replace('\\', "/") == target) + }) + .map(|p| format!("pid {}", p.pid)) + .collect(); + hitting.join(", ") +} + /// Build the transient work-state system message, or `None` when disabled or /// when there is no state to show yet. The caller pushes it as the LAST message /// before the model request and pops it right after, so it never reaches the @@ -1108,6 +1218,8 @@ async fn main() { intercom: IntercomBus::new(), subagent_runs: Mutex::new(std::collections::HashMap::new()), pending_oauth: Mutex::new(None), + peers: Mutex::new(Vec::new()), + last_concurrency_note: Mutex::new(None), }); // Apply disabled plugin list from config. @@ -1187,6 +1299,66 @@ async fn main() { }); } + // Cross-session presence: publish this session's rolling work-state so other + // processes in the SAME workspace can detect concurrent activity (and stop + // "fixing" phantom errors caused by a neighbor's in-flight edits). Per-pid + // JSON file under ~/.config/catalyst-code/presence//, rewritten + // every few seconds; stale records reaped by readers. Awareness only — no + // coordination/locking. The `workspace_activity` tool + the anomaly nudge + // in `run_turn` consume this; the cached peer snapshot avoids a filesystem + // read on every tool result. + { + let st = state.clone(); + let pid = std::process::id(); + let started = presence::unix_now(); + let presence_ws = { + let cfg = state.cfg.read().await; + cfg.workspace.clone() + }; + // Publish immediately so a peer checking right after we start sees us. + { + let ws = st.work_state.lock().await; + let session_id = st + .cfg + .read() + .await + .session_file + .as_ref() + .and_then(|p| p.file_name()) + .and_then(|n| n.to_str()) + .map(String::from); + let model = st.last_model.lock().await.clone(); + let rec = + presence::PresenceRecord::from_work_state(&ws, pid, session_id, model, started); + drop(ws); + presence::write_presence(&presence_ws, pid, &rec); + } + tokio::spawn(async move { + let interval = std::time::Duration::from_secs(8); + loop { + tokio::time::sleep(interval).await; + let ws = st.work_state.lock().await; + let session_id = st + .cfg + .read() + .await + .session_file + .as_ref() + .and_then(|p| p.file_name()) + .and_then(|n| n.to_str()) + .map(String::from); + let model = st.last_model.lock().await.clone(); + let rec = + presence::PresenceRecord::from_work_state(&ws, pid, session_id, model, started); + drop(ws); + presence::write_presence(&presence_ws, pid, &rec); + // Refresh the cached peer snapshot so the anomaly nudge stays + // current without a filesystem read on the hot path. + *st.peers.lock().await = presence::read_peers(&presence_ws, pid); + } + }); + } + let stdin = tokio::io::stdin(); let mut lines = BufReader::new(stdin).lines(); @@ -1220,6 +1392,7 @@ async fn main() { .with("providers", json!(cfg.provider_names())) .with("providerPresets", json!(provider_presets_json(&cfg))) .with("bash_timeout_secs", json!(cfg.bash_timeout_secs)) + .with("auto_compact", json!(cfg.auto_compact)) .with("resumed_messages", json!(conv_len)), ); // Tell the user when the harness staged its global defaults @@ -1595,12 +1768,22 @@ async fn main() { emit(&Event::new("approval_changed").with("mode", json!(new.as_str()))); } Command::SetConfig { key, value } => { - // ponytail: minimal runtime knob setter for the two values the - // TUI settings modal edits. Coerce string-or-number to u64. + // Minimal runtime knob setter for the values the TUI settings + // modal edits. Coerce string-or-number to u64, string-or-bool + // to bool. let as_u64 = |v: &Value| { v.as_u64() .or_else(|| v.as_str().and_then(|s| s.parse::().ok())) }; + let as_bool = |v: &Value| { + v.as_bool().or_else(|| { + v.as_str().and_then(|s| match s { + "1" | "true" | "on" => Some(true), + "0" | "false" | "off" => Some(false), + _ => None, + }) + }) + }; let mut cfg = state.cfg.write().await; let out_key = key.clone(); let mut out_val = value.clone(); @@ -1611,6 +1794,12 @@ async fn main() { out_val = json!(n); } } + "auto_compact" => { + if let Some(b) = as_bool(&value) { + cfg.auto_compact = b; + out_val = json!(b); + } + } _ => { drop(cfg); emit( @@ -1669,15 +1858,17 @@ async fn main() { clear_work_state(&state).await; emit(&Event::new("reset")); // TUI clears blocks; core keeps the trimmed conv } - Command::Compact => { - // Force compaction now, then emit a compacted event. + Command::Compact { instructions } => { + // Force compaction now, then emit a compacted event. Uses the + // summarize strategy (honoring any `/compact ` + // override or the configured `compact_instructions`) when an api + // key is present; falls back to naive drop-oldest otherwise. let mut messages = state.conversation.lock().await.clone(); if messages.len() > 2 { dispatch_lifecycle(&state, "pre_compact").await; let before_est = estimate_messages_tokens(&messages); // Size the reclaim against the user's actual model window, - // not a hardcoded 200k — and let compact_conversation digest - // oversized tool results when the tail alone is too big. + // not a hardcoded 200k. let model_ctx = { let last = state.last_model.lock().await.clone(); let models = state.models.read().await; @@ -1686,7 +1877,44 @@ async fn main() { .map(|m| m.context_window as u64) .unwrap_or(200_000) }; - compact_conversation(&mut messages, model_ctx); + emit( + &Event::new("compacting") + .with("before_tokens", json!(before_est)) + .with("trigger", json!("manual")), + ); + let cfg = state.cfg.read().await.clone(); + let model_name = state.last_model.lock().await.clone().unwrap_or_default(); + let rp = state.resolve_provider_for_model(&model_name).await; + // A `/compact ` override takes precedence over + // the configured default; empty/whitespace falls back. + let instr = match instructions + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + { + Some(s) => Some(s), + None => cfg.compact_instructions.as_deref(), + }; + // Manual compact is a one-shot — a fresh (never-cancelled) + // token is fine; there's no in-flight turn to abort it. + let cancel = CancellationToken::new(); + let summary_chars = if rp.api_key.is_some() && !model_name.is_empty() { + compact_with_summary( + &client, + &cfg, + &rp, + &model_name, + &mut messages, + &cancel, + false, + model_ctx, + instr, + ) + .await + } else { + compact_conversation(&mut messages, model_ctx); + 0 + }; *state.conversation.lock().await = messages.clone(); let after_est = estimate_messages_tokens(&messages); *state.estimated_tokens.lock().await = after_est; @@ -1698,7 +1926,8 @@ async fn main() { emit( &Event::new("compacted") .with("before_tokens", json!(before_est)) - .with("after_tokens", json!(after_est)), + .with("after_tokens", json!(after_est)) + .with("summary_chars", json!(summary_chars)), ); } else { emit(&Event::new("info").with("message", json!("nothing to compact yet"))); @@ -1912,6 +2141,88 @@ async fn main() { .with("session_file", json!(session_file)), ); } + Command::Context => { + // Token-breakdown: where is the context window being spent? + // Aggregates per-message token estimates (same char/4 heuristic + // the footer uses) so the user can see the biggest consumers + // before compaction fires. Read-only — never mutates state. + let conv = state.conversation.lock().await.clone(); + let total = { + let last_real = *state.last_real_prompt_tokens.lock().await; + let len_at = *state.conv_len_at_last_real.lock().await; + grounded_estimate(&conv, last_real, len_at) + }; + let model_ctx = { + let last = state.last_model.lock().await.clone(); + let models = state.models.read().await; + last.as_deref() + .and_then(|m| models.iter().find(|mi| mi.id == m)) + .map(|m| m.context_window as u64) + .unwrap_or(200_000) + }; + let pct = if model_ctx > 0 { + (total as f64 / model_ctx as f64 * 100.0).round() as u64 + } else { + 0 + }; + // Per-message estimates; role buckets are aggregated below from + // the entries for clean u64 values. + let mut entries: Vec = Vec::with_capacity(conv.len()); + for (i, m) in conv.iter().enumerate() { + let tokens = estimate_messages_tokens(std::slice::from_ref(m)); + let role = m.role(); + let preview: String = m + .content_text() + .map(|t| { + let t = t.replace('\n', " "); + if t.chars().count() > 100 { + format!("{}…", t.chars().take(100).collect::()) + } else { + t + } + }) + .unwrap_or_else(|| "(no text / multimodal)".to_string()); + entries.push(json!({ + "index": i, + "role": role, + "tokens": tokens, + "preview": preview, + })); + } + // Aggregate per-role token totals. + let role_obj: Value = { + let mut counts: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + for e in &entries { + let r = e["role"].as_str().unwrap_or("").to_string(); + let t = e["tokens"].as_u64().unwrap_or(0); + *counts.entry(r).or_insert(0) += t; + } + let mut map = serde_json::Map::new(); + for (k, v) in counts { + map.insert(k, json!(v)); + } + Value::Object(map) + }; + let system_tokens = entries + .iter() + .filter(|e| e["role"].as_str() == Some("system")) + .map(|e| e["tokens"].as_u64().unwrap_or(0)) + .sum::(); + // Top 10 consumers by tokens (descending). + entries.sort_by(|a, b| b["tokens"].as_u64().cmp(&a["tokens"].as_u64())); + let top: Vec = entries.iter().take(10).cloned().collect(); + emit( + &Event::new("context_breakdown") + .with("total_tokens", json!(total)) + .with("context_window", json!(model_ctx)) + .with("pct", json!(pct)) + .with("messages", json!(conv.len())) + .with("system_tokens", json!(system_tokens)) + .with("by_role", role_obj) + .with("top_consumers", json!(top)), + ); + } Command::InstallPlugin { path } => { let dir = std::path::PathBuf::from(&path); match state.plugin_manager.install(&dir) { @@ -2312,6 +2623,14 @@ async fn main() { if let Some(h) = h { let _ = h.await; } + // Clean up our presence record so peers don't see a stale session. Best + // effort — a kill -9 / crash leaves a stale file that `read_peers` reaps + // by mtime, so this is an optimization (instant disappearance), not a + // correctness requirement. + { + let ws = state.cfg.read().await.workspace.clone(); + presence::clear_presence(&ws, std::process::id()); + } } /// Check if a tool call matches a permission rule. Used by the approval gate @@ -2473,6 +2792,130 @@ fn build_skill_prompt(skill: &subagent::SkillEntry, task: Option<&str>) -> Strin p } +/// Expand `@` file mentions in a prompt by inlining the referenced +/// file's contents directly, so the model sees them without a `read_file` +/// round-trip — mirroring how `apply_skill` inlines a skill body. The +/// transcript still shows the concise `@path` (the TUI/web logged the raw +/// text before the core received it); only the message the model reads is +/// expanded. +/// +/// A mention is `@` followed by a non-whitespace path, where the `@` is at +/// start-of-string or preceded by whitespace (so emails / `foo@bar` and +/// inline `@param` tags without a leading space don't trigger). Paths resolve +/// relative to the workspace; absolute paths (leading `/`) and `..`/`.` paths +/// are honored as-is — the core has unrestricted FS access, so `@../` and +/// `@/abs` reach outside the workspace (matching the TUI's mention completion). +/// Directories, files larger than `max_bytes`, and unreadable paths are left +/// as-is so the model can fall back to `read_file`. Returns the expanded +/// prompt and the list of paths successfully inlined. +fn expand_file_mentions( + prompt: &str, + workspace: &std::path::Path, + max_bytes: u64, +) -> (String, Vec) { + let chars: Vec<(usize, char)> = prompt.char_indices().collect(); + let mut out = String::with_capacity(prompt.len() + 256); + let mut attached: Vec = Vec::new(); + let mut k = 0; + let mut prev_ws_or_start = true; + while k < chars.len() { + let (idx, ch) = chars[k]; + if ch == '@' && prev_ws_or_start { + // Span from after '@' to the next whitespace char (or end). + let tok_byte_start = idx + '@'.len_utf8(); + let mut m = k + 1; + while m < chars.len() && !is_mention_ws(chars[m].1) { + m += 1; + } + let tok_byte_end = if m < chars.len() { + chars[m].0 + } else { + prompt.len() + }; + let raw = &prompt[tok_byte_start..tok_byte_end]; + if !raw.is_empty() { + if let Some((path, content)) = read_mentioned_file(raw, workspace, max_bytes) { + out.push('@'); + out.push_str(&path); + out.push_str("\n\n"); + out.push_str(&content); + if !content.ends_with('\n') { + out.push('\n'); + } + out.push_str("\n"); + attached.push(path); + k = m; + prev_ws_or_start = true; // the block ends in '\n' + continue; + } + } + // Not an attachable mention: emit the '@' and keep scanning. + out.push('@'); + k += 1; + prev_ws_or_start = false; + } else { + out.push(ch); + prev_ws_or_start = is_mention_ws(ch); + k += 1; + } + } + (out, attached) +} + +fn is_mention_ws(c: char) -> bool { + matches!(c, ' ' | '\t' | '\n' | '\r') +} + +/// Try to read a mentioned file. The raw token may carry trailing prose +/// punctuation ("see @file.rs." → "file.rs"); try the token verbatim first, +/// then with trailing punctuation stripped, so legitimate paths keep their +/// characters while common prose edge cases still resolve. +fn read_mentioned_file( + token: &str, + workspace: &std::path::Path, + max_bytes: u64, +) -> Option<(String, String)> { + let trimmed = token.trim_end_matches(|c: char| { + matches!( + c, + '.' | ',' | ';' | ':' | '!' | '?' | ')' | ']' | '}' | '\'' | '"' + ) + }); + for cand in [token, trimmed] { + if cand.is_empty() { + continue; + } + if let Some(res) = try_read_mentioned_file(cand, workspace, max_bytes) { + return Some(res); + } + } + None +} + +fn try_read_mentioned_file( + token: &str, + workspace: &std::path::Path, + max_bytes: u64, +) -> Option<(String, String)> { + let p = std::path::Path::new(token); + let resolved = if p.is_absolute() { + p.to_path_buf() + } else { + workspace.join(p) + }; + let meta = std::fs::metadata(&resolved).ok()?; + if meta.is_dir() { + return None; + } + if meta.len() > max_bytes { + return None; + } + let content = std::fs::read_to_string(&resolved).ok()?; + Some((token.to_string(), content)) +} + /// Start (or queue) an assistant turn for `prompt`. Shared by `send` and /// `apply_skill`: if a turn is already running, buffer this prompt one-deep /// (the running turn's drain picks it up); otherwise spawn run_turn_and_drain. @@ -2618,6 +3061,123 @@ pub(crate) async fn dispatch_lifecycle(st: &Arc, hook: &str) { } } +/// Run every enabled plugin's pre-execution hook for `hook_name` against a tool +/// call, composing each hook's `modify` into `exec_args` and recording reasons +/// into `hook_notes`. Returns `Some(deny_message)` when a hook denies the call +/// (the caller emits the tool_result and skips the tool), or `None` to proceed. +/// Used for BOTH the tool-specific pre_* hook (pre_bash/pre_write/pre_read) and +/// the catch-all `pre_tool` that fires for every tool call — giving a plugin the +/// same per-call reach over `memory`/`todo_write`/`git_*`/`subagent`/… that a +/// core edit of the dispatch loop has. +async fn run_pre_hooks( + st: &Arc, + cfg: &crate::config::Config, + hook_name: &str, + tool_name: &str, + exec_args: &mut Value, + hook_notes: &mut Vec, +) -> Option { + let configs = st.plugin_manager.get_hook_configs(hook_name); + if configs.is_empty() { + return None; + } + let session_id = cfg + .session_file + .as_ref() + .map(|p| p.display().to_string()) + .unwrap_or_default(); + let ws = cfg.workspace.display().to_string(); + for (plugin_name, config) in &configs { + let ctx = plugins::build_context( + hook_name, + tool_name, + &ws, + Some(exec_args), + &session_id, + config.pass_args, + ); + let result = plugins::execute_hook(hook_name, plugin_name, config, &ctx).await; + if !result.allow { + return Some(format!( + "tool call '{}' denied by plugin '{}' hook '{}': {}", + tool_name, plugin_name, hook_name, result.reason + )); + } + if let Some(ref modify) = result.modify { + plugins::apply_modify(exec_args, modify); + } + if !result.reason.is_empty() { + hook_notes.push(format!("{}/{}: {}", plugin_name, hook_name, result.reason)); + } + } + None +} + +/// Run every enabled plugin's post-execution hook for `hook_name`, handing each +/// the tool's CURRENT result (so it can read it) and letting it MODIFY that +/// result. A post hook returns `modify: { "output": "…", "ok": false }` to +/// replace the result text / flip success — e.g. redact a secret, append +/// context, or reformat. Post hooks never block (the op already ran), so +/// `allow:false` is ignored (only its `reason` is surfaced). Used for BOTH the +/// tool-specific post_* hook and the catch-all `post_tool`. +async fn run_post_hooks( + st: &Arc, + cfg: &crate::config::Config, + hook_name: &str, + tool_name: &str, + exec_args: &Value, + outcome: &mut tools::Outcome, + hook_notes: &mut Vec, +) { + let configs = st.plugin_manager.get_hook_configs(hook_name); + if configs.is_empty() { + return; + } + let session_id = cfg + .session_file + .as_ref() + .map(|p| p.display().to_string()) + .unwrap_or_default(); + let ws = cfg.workspace.display().to_string(); + for (plugin_name, config) in &configs { + // Give the hook the current result so it can redact/append/transform it. + let result_json = json!({ + "ok": outcome.ok, + "output": outcome.output, + "diff": outcome.diff, + }); + let mut ctx = plugins::build_context( + hook_name, + tool_name, + &ws, + Some(exec_args), + &session_id, + config.pass_args, + ); + if let Some(obj) = ctx.as_object_mut() { + obj.insert("result".to_string(), result_json); + } + let result = plugins::execute_hook(hook_name, plugin_name, config, &ctx).await; + // Post hooks can't block; a deny is treated as an observed note only. + if !result.reason.is_empty() { + hook_notes.push(format!("{}/{}: {}", plugin_name, hook_name, result.reason)); + } + // Apply an optional result mutation: `output` replaces the text, `ok` + // flips success, `diff` (string) replaces / (null) clears the diff. + if let Some(obj) = result.modify.as_ref().and_then(|m| m.as_object()) { + if let Some(out) = obj.get("output").and_then(|v| v.as_str()) { + outcome.output = out.to_string(); + } + if let Some(ok) = obj.get("ok").and_then(|v| v.as_bool()) { + outcome.ok = ok; + } + if let Some(diff) = obj.get("diff") { + outcome.diff = diff.as_str().map(String::from); + } + } + } +} + /// Token counts reported in the `metrics` event, which drive the footer's /// context budget. The provider returns the *last request's* usage /// (prompt/completion tokens ≈ the live context size) when the endpoint @@ -2789,11 +3349,29 @@ async fn run_turn( // Ensure system prompt is present; persist every finalized message to the session file. let mut init_est_add = 0u64; + // Expand `@` file mentions so the model sees the referenced file's + // contents directly (no `read_file` round-trip) — mirroring how + // `apply_skill` inlines a skill body. The transcript keeps the concise + // `@path` the user typed (the TUI/web already logged the raw text). + let (user_text, attached_files) = { + let c = st.cfg.read().await; + expand_file_mentions(&prompt, &c.workspace, c.max_read_bytes) + }; + if !attached_files.is_empty() { + emit(&Event::new("info").with( + "message", + json!(format!( + "attached {} file(s) from @mentions: {}", + attached_files.len(), + attached_files.join(", ") + )), + )); + } { let mut conv = st.conversation.lock().await; if conv.is_empty() { let workspace = st.cfg.read().await.workspace.clone(); - let sys_msg = Message::system(build_system_prompt(&workspace, true)); + let sys_msg = Message::system(build_main_system_prompt(&workspace, &st.plugin_manager)); init_est_add += estimate_message_tokens(&sys_msg); conv.push(sys_msg); if let Some(p) = st.cfg.read().await.session_file.as_ref() { @@ -2806,7 +3384,7 @@ async fn run_turn( let user_msg = match (&images, allow_vision) { (Some(imgs), true) if !imgs.is_empty() => { let mut parts: Vec = vec![ContentPart::Text { - text: prompt.clone(), + text: user_text.clone(), }]; for img in imgs { let url = image_to_data_url(img); @@ -2816,7 +3394,7 @@ async fn run_turn( } Message::user_multimodal(parts) } - _ => Message::user(prompt.clone()), + _ => Message::user(user_text.clone()), }; init_est_add += estimate_message_tokens(&user_msg); conv.push(user_msg); @@ -2944,10 +3522,21 @@ async fn run_turn( // Main agent tool list: built-in tools (minus the subagent-only intercom // coordination tools contact_supervisor/intercom, registered only inside - // child runs) MERGED with tools declared by enabled plugins. Plugin tools - // that collide with a built-in (or an already-registered plugin tool) name - // are skipped — a plugin can never shadow a core tool, and the first plugin - // to claim a name wins (matching the project>global plugin override model). + // child runs) MERGED with tools declared by enabled plugins, then filtered + // by every plugin's `disable_tools`. Three plugin capabilities converge + // here, mirroring what a direct core edit can do to the tool list: + // • ADD — a plugin `tools` entry adds a new capability. + // • OVERRIDE — a plugin tool with `override:true` whose name matches a + // built-in REPLACES that built-in: the plugin's declared + // schema is shown to the model and calls route to the + // plugin handler (see the dispatch below). + // • REMOVE — `disable_tools` names are dropped from the final list + // (built-in OR override). `disable_tools` is the strongest + // lever: a disabled name is gone, period. + // A plugin tool that merely collides with a built-in name (no `override`) + // is still skipped — the built-in wins, unchanged. + let overridden = st.plugin_manager.overridden_tool_names(); + let disabled = st.plugin_manager.disabled_tools(); let mut reserved: std::collections::HashSet = std::collections::HashSet::new(); reserved.insert("contact_supervisor".into()); reserved.insert("intercom".into()); @@ -2960,7 +3549,9 @@ async fn run_turn( .and_then(|v| v.as_str()) .unwrap_or(""); reserved.insert(n.to_string()); - n != "contact_supervisor" && n != "intercom" + // Hide the reserved subagent-only tools, AND any built-in a plugin + // is overriding (its plugin version is added below instead). + n != "contact_supervisor" && n != "intercom" && !overridden.contains(n) }) .collect(); for d in st.plugin_manager.tool_definitions() { @@ -2969,7 +3560,11 @@ async fn run_turn( .and_then(|f| f.get("name")) .and_then(|v| v.as_str()) .unwrap_or(""); - if !reserved.insert(n.to_string()) { + // An override tool replaces the built-in (already excluded above), so + // it's always added and claims the name. A plain custom tool is added + // only if its name isn't already taken (built-in or another plugin). + let is_override = overridden.contains(n); + if !is_override && !reserved.insert(n.to_string()) { eprintln!( "[plugins] tool '{}' collides with a built-in or already-registered tool; skipping", n @@ -2978,6 +3573,17 @@ async fn run_turn( } tool_defs.push(d); } + // REMOVE: `disable_tools` is a final, composition-winning filter — a + // disabled name vanishes whether it was a built-in, an override, or a + // custom plugin tool. (No-op when no plugin disables anything.) + if !disabled.is_empty() { + tool_defs.retain(|d| { + d.get("function") + .and_then(|f| f.get("name")) + .and_then(|v| v.as_str()) + .is_none_or(|n| !disabled.contains(n)) + }); + } let mut timer = TurnTimer::new(); // Idle compaction: if 60+ minutes since the last turn completed, compact the @@ -2985,11 +3591,17 @@ async fn run_turn( // as the threshold path; falls back to naive drop-oldest without an api key. { let last = *st.last_turn_time.lock().await; - if last.elapsed().as_secs() > 3600 { + let auto_compact = st.cfg.read().await.auto_compact; + if auto_compact && last.elapsed().as_secs() > 3600 { let mut messages = st.conversation.lock().await.clone(); if messages.len() > 4 { - dispatch_lifecycle(st, "pre_compact").await; let est = { *st.estimated_tokens.lock().await }; + emit( + &Event::new("compacting") + .with("before_tokens", json!(est)) + .with("trigger", json!("idle")), + ); + dispatch_lifecycle(st, "pre_compact").await; let cfg = st.cfg.read().await.clone(); let rp = st.resolve_provider_for_model(&model).await; let idle_ctx = st @@ -3010,6 +3622,7 @@ async fn run_turn( &cancel, false, idle_ctx, + cfg.compact_instructions.as_deref(), ) .await } else { @@ -3081,9 +3694,13 @@ async fn run_turn( let cfg = st.cfg.read().await.clone(); // Context window management: compact once past the configured threshold - // (default 70%). The 95% hard cap is a floor — compact by then even if the + // (default 90%). The 95% hard cap is a floor — compact by then even if the // configured threshold is higher, and force the summarize strategy even // when disabled (naive drop-oldest may not reclaim enough at critical capacity). + // `auto_compact` (default true) gates ALL automatic compaction (this threshold + // path + the idle-compaction block below). When false, no compaction fires + // automatically — the user must /compact manually (or /clear). Mirrors Claude + // Code's autoCompactEnabled / DISABLE_AUTO_COMPACT. let mut messages = st.conversation.lock().await.clone(); let (model_ctx, thinking_levels, max_tokens) = st .models @@ -3147,8 +3764,13 @@ async fn run_turn( ); } } - if est > threshold.min(hard_cap) && messages.len() > 4 { + if cfg.auto_compact && est > threshold.min(hard_cap) && messages.len() > 4 { let force_summarize = est > hard_cap; + emit( + &Event::new("compacting") + .with("before_tokens", json!(est)) + .with("trigger", json!("threshold")), + ); dispatch_lifecycle(st, "pre_compact").await; let summary_chars = compact_with_summary( client, @@ -3159,6 +3781,7 @@ async fn run_turn( &cancel, force_summarize, model_ctx, + cfg.compact_instructions.as_deref(), ) .await; *st.conversation.lock().await = messages.clone(); @@ -3372,19 +3995,16 @@ async fn run_turn( } }; - // Approval gate for destructive tools. Plugin-declared tools - // carry their own kind (default Destructive); built-ins use - // the static classify() table. A plugin tool that collides - // with a built-in name is hidden from the model's tool list - // (the merge gives built-ins precedence), and `is_builtin` - // here ensures its kind never overrides the built-in's either. + // Approval gate for destructive tools. A plugin tool that + // dispatches (a custom name, OR an `override:true` tool that + // replaces a built-in) carries its own kind; everything else + // uses the static classify() table. A plugin tool that merely + // collides with a built-in name (no override) does NOT + // dispatch to the plugin, so it falls through to classify(). let cfg = st.cfg.read().await.clone(); - let kind = if tools::is_builtin(&name) { - tools::classify(&name) - } else { - st.plugin_manager - .tool_kind(&name) - .unwrap_or_else(|| tools::classify(&name)) + let kind = match st.plugin_manager.tool_config(&name) { + Some(tc) if tc.override_builtin || !tools::is_builtin(&name) => tc.kind, + _ => tools::classify(&name), }; let kind_str: &'static str = match kind { tools::ToolKind::ReadOnly => "readonly", @@ -3482,89 +4102,69 @@ async fn run_turn( } } - // Dispatch pre-execution hooks for this tool. Each enabled - // plugin registered for this hook point runs exactly once, in - // order. A hook may: allow (optionally overriding specific arg - // fields via `modify`, and/or posting a `reason` the model will - // see), or deny (the tool call is skipped and the reason is - // returned to the model). Hooks compose: each sees the args as - // amended by earlier hooks. + // Dispatch pre-execution hooks for this tool. Two phases compose: + // 1. the tool-SPECIFIC pre_* hook (pre_bash/pre_write/pre_read) + // — transforms/audits/denies that tool's call; and + // 2. the catch-all `pre_tool` hook, which fires for EVERY tool + // (memory, todo_write, git_*, subagent, plugin tools, …) + // so a plugin can intercept any call — the same reach a + // core edit of this dispatch loop has. pre_tool runs AFTER + // the specific hook so it sees the final amended args. + // Each hook may allow (optionally overriding arg fields via + // `modify`, and/or posting a `reason`), or deny (the call is + // skipped and the reason is returned to the model). Hooks + // compose: each sees the args as amended by earlier hooks. let hook_name = match name.as_str() { "bash" => "pre_bash", "write_file" | "edit" => "pre_write", "read_file" | "grep" | "glob" => "pre_read", _ => "", }; - let pre_configs = if hook_name.is_empty() { - Vec::new() - } else { - st.plugin_manager.get_hook_configs(hook_name) - }; - // exec_args starts as the original args and is amended in - // place by pre-hooks. Only clone when hooks will actually run, - // so large write payloads aren't copied in the common case. - let mut exec_args = if pre_configs.is_empty() { - args - } else { - args.clone() - }; + let any_pre = (!hook_name.is_empty() && st.plugin_manager.has_hook(hook_name)) + || st.plugin_manager.has_hook("pre_tool"); + // exec_args starts as the original args and is amended in place + // by pre-hooks. Only clone when a hook will actually run, so + // large write payloads aren't copied in the common case. + let mut exec_args = if any_pre { args.clone() } else { args }; let mut hook_notes: Vec = Vec::new(); - let mut denied_by_hook = false; - for (plugin_name, config) in &pre_configs { - let session_id = cfg - .session_file - .as_ref() - .map(|p| p.display().to_string()) - .unwrap_or_default(); - let ctx = plugins::build_context( + let mut denied: Option = None; + if !hook_name.is_empty() { + denied = run_pre_hooks( + st, + &cfg, hook_name, &name, - &cfg.workspace.display().to_string(), - Some(&exec_args), - &session_id, - config.pass_args, + &mut exec_args, + &mut hook_notes, + ) + .await; + } + if denied.is_none() && name != "finish" { + denied = run_pre_hooks( + st, + &cfg, + "pre_tool", + &name, + &mut exec_args, + &mut hook_notes, + ) + .await; + } + if let Some(msg) = denied { + emit( + &Event::new("tool_result") + .with("id", json!(id)) + .with("ok", json!(false)) + .with("output", json!(msg)), ); - let result = - plugins::execute_hook(hook_name, plugin_name, config, &ctx).await; - if !result.allow { - // Deny: skip the tool call and tell the model why. - let msg = format!( - "tool call '{}' denied by plugin '{}' hook '{}': {}", - name, plugin_name, hook_name, result.reason - ); - emit( - &Event::new("tool_result") - .with("id", json!(id)) - .with("ok", json!(false)) - .with("output", json!(msg)), - ); - let tool_result = Message::tool(id.clone(), msg); - let est = estimate_message_tokens(&tool_result); - let mut conv = st.conversation.lock().await; - conv.push(tool_result); - if let Some(p) = st.cfg.read().await.session_file.as_ref() { - session::append(p, conv.last().unwrap()); - } - *st.estimated_tokens.lock().await += est; - denied_by_hook = true; - break; - } - // Allow: merge `modify` over the running args so a hook - // can override specific fields (e.g. reformatted `content` - // or a fixed `command`) without dropping the rest (e.g. - // `path`, `edits`). The contract is "return only the keys - // you want to change"; anything else is preserved. - if let Some(ref modify) = result.modify { - plugins::apply_modify(&mut exec_args, modify); - } - // Remember non-empty reasons so the model is told its tool - // call was inspected/modified (and can react accordingly). - if !result.reason.is_empty() { - hook_notes - .push(format!("{}/{}: {}", plugin_name, hook_name, result.reason)); + let tool_result = Message::tool(id.clone(), msg); + let est = estimate_message_tokens(&tool_result); + let mut conv = st.conversation.lock().await; + conv.push(tool_result); + if let Some(p) = st.cfg.read().await.session_file.as_ref() { + session::append(p, conv.last().unwrap()); } - } - if denied_by_hook { + *st.estimated_tokens.lock().await += est; continue; } @@ -3658,7 +4258,32 @@ async fn run_turn( // The async ones are wrapped in a `select!` on the turn cancel // so /abort can interrupt them mid-flight — kill_on_drop frees // the spawned child when the future is dropped. - let mut outcome = if name == "bash" { + let mut outcome = if let Some(tc) = st + .plugin_manager + .tool_config(&name) + .filter(|tc| tc.override_builtin || !tools::is_builtin(&name)) + { + // Plugin-declared tool: dispatch to its handler script + // (subprocess, stdin=args JSON, stdout={ok,output}). + // This branch covers BOTH custom plugin tools (a name no + // built-in owns) AND `override:true` tools that REPLACE + // a built-in's implementation — the filter admits a + // built-in name only when the plugin explicitly opted + // into overriding it, so a mere name collision still + // falls through to the built-in handler below. Wrapped in + // a select! on the turn cancel so /abort can interrupt it + // mid-flight; kill_on_drop frees the child. + let session_id = cfg + .session_file + .as_ref() + .map(|p| p.display().to_string()) + .unwrap_or_default(); + let ws = cfg.workspace.display().to_string(); + tokio::select! { + o = plugins::execute_plugin_tool(&name, &tc, &exec_args, &ws, &session_id) => o, + _ = cancel.cancelled() => tools::Outcome::err(format!("{name} aborted")), + } + } else if name == "bash" { let cmd = exec_args .get("command") .and_then(|v| v.as_str()) @@ -3717,28 +4342,6 @@ async fn run_turn( return; } } - } else if let Some(tc) = st - .plugin_manager - .tool_config(&name) - .filter(|_| !tools::is_builtin(&name)) - { - // Plugin-declared tool: dispatch to its handler script - // (subprocess, stdin=args JSON, stdout={ok,output}). The - // `is_builtin` guard means a plugin tool that collides - // with a built-in name can never hijack it (the built-in - // is always routed to its own handler below). Wrapped in - // a select! on the turn cancel so /abort can interrupt it - // mid-flight; kill_on_drop frees the child. - let session_id = cfg - .session_file - .as_ref() - .map(|p| p.display().to_string()) - .unwrap_or_default(); - let ws = cfg.workspace.display().to_string(); - tokio::select! { - o = plugins::execute_plugin_tool(&name, &tc, &exec_args, &ws, &session_id) => o, - _ = cancel.cancelled() => tools::Outcome::err(format!("{name} aborted")), - } } else { tools::execute(&name, &exec_args, &cfg) }; @@ -3773,7 +4376,15 @@ async fn run_turn( } } - // Dispatch post-execution hooks for this tool. + // Dispatch post-execution hooks for this tool. Two phases, + // mirroring the pre-hook structure: the tool-SPECIFIC post_* + // hook (post_bash/post_write/post_read), then the catch-all + // `post_tool` that fires for EVERY tool. Each hook receives the + // tool's CURRENT result and may MODIFY it (return + // `modify: {"output":…, "ok":…, "diff":…}`) — e.g. redact a + // secret, append context, reformat. Post-hooks never block (the + // op already ran); `allow:false` is ignored, only `reason` + + // `modify` are honored. let post_hook = match name.as_str() { "bash" => "post_bash", "write_file" | "edit" => "post_write", @@ -3781,32 +4392,28 @@ async fn run_turn( _ => "", }; if !post_hook.is_empty() { - let configs = st.plugin_manager.get_hook_configs(post_hook); - for (plugin_name, config) in &configs { - let session_id = cfg - .session_file - .as_ref() - .map(|p| p.display().to_string()) - .unwrap_or_default(); - let ctx = plugins::build_context( - post_hook, - &name, - &cfg.workspace.display().to_string(), - Some(&exec_args), - &session_id, - config.pass_args, - ); - // Post-hooks can't block (the op already ran), but their - // reason is surfaced to the model as a note. - let result = - plugins::execute_hook(post_hook, plugin_name, config, &ctx).await; - if !result.reason.is_empty() { - hook_notes.push(format!( - "{}/{}: {}", - plugin_name, post_hook, result.reason - )); - } - } + run_post_hooks( + st, + &cfg, + post_hook, + &name, + &exec_args, + &mut outcome, + &mut hook_notes, + ) + .await; + } + if name != "finish" { + run_post_hooks( + st, + &cfg, + "post_tool", + &name, + &exec_args, + &mut outcome, + &mut hook_notes, + ) + .await; } // finish sentinel: the model signaled completion. @@ -3873,6 +4480,19 @@ async fn run_turn( outcome.output.push_str("\n\nPlugin hooks:\n- "); outcome.output.push_str(&hook_notes.join("\n- ")); } + // Cross-session anomaly nudge: if another session is + // active in this workspace and this tool failed (or touched + // a file a peer is editing), append a note so the agent + // checks the neighbors before assuming it caused the error. + // Uses the cached peer snapshot — no filesystem read here. + if let Some(note) = + maybe_concurrency_note(st, &name, &exec_args, outcome.ok).await + { + outcome.output.push_str("\n\n"); + outcome.output.push_str(¬e); + } + // Debug log: records full tool args (file contents, commands) which may + // include secrets the model handles. Opt-in (cfg.debug_log), user-owned. st.logger.log("tool", json!({ "name": name, "args": args_str, "ok": outcome.ok, "output_len": outcome.output.len() })); let mut ev = Event::new("tool_result") .with("id", json!(id)) @@ -4546,7 +5166,7 @@ fn truncate_str(s: &str, n: usize) -> String { async fn refresh_memory_injection(state: &State) -> String { let ws = state.cfg.read().await.workspace.clone(); let mem = memory_injection(&ws, ""); - let new_system = build_system_prompt(&ws, true); + let new_system = build_main_system_prompt(&ws, &state.plugin_manager); let mut conv = state.conversation.lock().await; if let Some(first) = conv.first() { let old_content = first.content_text().unwrap_or(""); @@ -4666,6 +5286,7 @@ pub async fn compact_with_summary( cancel: &CancellationToken, force_summarize: bool, context_window: u64, + instructions: Option<&str>, ) -> usize { // Returns the character count of the produced summary system message (0 // when no summary was generated — naive drop-oldest fallback or a @@ -4685,7 +5306,8 @@ pub async fn compact_with_summary( } let to_summarize: Vec = messages[1..tail_start].to_vec(); let kept: Vec = messages[tail_start..].to_vec(); - let summary = provider::summarize(client, provider, model, &to_summarize, cancel).await; + let summary = + provider::summarize(client, provider, model, &to_summarize, cancel, instructions).await; let mut summary_chars = 0usize; let mut compacted = vec![messages[0].clone()]; if let Some(s) = summary { @@ -5274,6 +5896,44 @@ mod work_state_tests { // Most-recent (f11) is at the front. assert_eq!(ws.recent_files[0], "f11.rs"); } + + #[test] + fn peers_touching_matches_exact_normalized_path() { + let mk = |pid: u32, files: &[&str]| presence::PresenceRecord { + pid, + session_id: None, + started_at: 0, + last_heartbeat: 0, + goal: String::new(), + in_progress: vec![], + next: vec![], + recent_files: files.iter().map(|s| s.to_string()).collect(), + last_activity: String::new(), + model: None, + }; + let peers = vec![mk(111, &["core/src/main.rs"]), mk(222, &["other.go"])]; + // exact match → the touching peer's pid + assert_eq!( + peers_touching(&peers, "edit", &json!({"path":"core/src/main.rs"})), + "pid 111" + ); + // separator-normalized (backslash) still matches + assert_eq!( + peers_touching(&peers, "write_file", &json!({"path":"core\\src\\main.rs"})), + "pid 111" + ); + // a path nobody is touching → empty (no false positive) + assert_eq!( + peers_touching(&peers, "read_file", &json!({"path":"foo.rs"})), + "" + ); + // a non-file tool (bash) → empty + assert_eq!(peers_touching(&peers, "bash", &json!({"command":"ls"})), ""); + // multiple touching peers → comma-list + let peers2 = vec![mk(111, &["shared.rs"]), mk(333, &["shared.rs"])]; + let s = peers_touching(&peers2, "edit", &json!({"path":"shared.rs"})); + assert!(s.contains("pid 111") && s.contains("pid 333")); + } } #[cfg(test)] @@ -5542,3 +6202,130 @@ mod ask_tests { assert!(!out.contains("(skipped)")); } } + +#[cfg(test)] +mod expand_mentions_tests { + use super::*; + + fn fresh_workspace() -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!( + "catalyst-code-mentions-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&d).unwrap(); + d + } + + #[test] + fn inlines_existing_file() { + let ws = fresh_workspace(); + std::fs::write(ws.join("main.rs"), "fn main() {}\n").unwrap(); + let (out, attached) = expand_file_mentions("fix @main.rs please", &ws, u64::MAX); + assert_eq!(attached, vec!["main.rs".to_string()]); + assert!(out.contains("")); + assert!(out.contains("fn main() {}")); + assert!(out.contains("")); + // The surrounding prose is preserved (no trailing newline in input). + assert!(out.starts_with("fix ")); + assert!(out.ends_with(" please")); + } + + #[test] + fn leaves_missing_path_as_is() { + let ws = fresh_workspace(); + let (out, attached) = expand_file_mentions("look at @nope.rs", &ws, u64::MAX); + assert!(attached.is_empty()); + assert_eq!(out, "look at @nope.rs"); + } + + #[test] + fn email_not_triggered() { + // `foo@bar` has no whitespace before `@`, so it must NOT be a mention. + let ws = fresh_workspace(); + std::fs::write(ws.join("bar"), "x").unwrap(); + let (out, attached) = expand_file_mentions("email foo@bar.com here", &ws, u64::MAX); + assert!(attached.is_empty()); + assert_eq!(out, "email foo@bar.com here"); + } + + #[test] + fn inline_param_tag_not_triggered_without_space() { + // `@param` embedded mid-word (no leading space) is left alone even if a + // file named `param` exists. + let ws = fresh_workspace(); + std::fs::write(ws.join("param"), "x").unwrap(); + let (out, attached) = expand_file_mentions("see the@param tag", &ws, u64::MAX); + assert!(attached.is_empty()); + assert_eq!(out, "see the@param tag"); + } + + #[test] + fn strips_trailing_punctuation() { + let ws = fresh_workspace(); + std::fs::write(ws.join("file.rs"), "pub fn f() {}\n").unwrap(); + let (out, attached) = expand_file_mentions("see @file.rs.", &ws, u64::MAX); + assert_eq!(attached, vec!["file.rs".to_string()]); + assert!(out.contains("")); + // A file with a trailing dot literally does not exist, so the literal + // candidate is skipped and the trimmed one wins. + assert!(!out.contains("")); + } + + #[test] + fn skips_directory() { + let ws = fresh_workspace(); + std::fs::create_dir_all(ws.join("sub")).unwrap(); + let (out, attached) = expand_file_mentions("look at @sub", &ws, u64::MAX); + assert!(attached.is_empty()); + // Directory left as-is so the model can fall back to read_file/list_dir. + assert_eq!(out, "look at @sub"); + } + + #[test] + fn skips_oversized_file() { + let ws = fresh_workspace(); + // max_bytes = 3, file is 10 bytes → skipped, left as-is. + std::fs::write(ws.join("big.txt"), "0123456789").unwrap(); + let (out, attached) = expand_file_mentions("@big.txt", &ws, 3); + assert!(attached.is_empty()); + assert_eq!(out, "@big.txt"); + } + + #[test] + fn multiple_mentions_inlined() { + let ws = fresh_workspace(); + std::fs::write(ws.join("a.rs"), "a\n").unwrap(); + std::fs::write(ws.join("b.go"), "b\n").unwrap(); + let (out, attached) = expand_file_mentions("@a.rs and @b.go", &ws, u64::MAX); + assert_eq!(attached, vec!["a.rs".to_string(), "b.go".to_string()]); + assert!(out.contains("")); + assert!(out.contains("")); + } + + #[cfg(unix)] + #[test] + fn absolute_path_inlined() { + // Absolute paths are honored (core has unrestricted FS access), even + // though they lie outside the workspace. + let dir = std::env::temp_dir().join(format!( + "catalyst-code-abs-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + let f = dir.join("abs.txt"); + std::fs::write(&f, "abs content\n").unwrap(); + let ws = fresh_workspace(); + let mention = format!("@{}", f.display()); + let (out, attached) = expand_file_mentions(&mention, &ws, u64::MAX); + assert_eq!(attached.len(), 1); + assert!(out.contains("abs content")); + assert!(out.contains(" { @@ -636,27 +643,13 @@ fn slugify(name: &str) -> String { out.trim_matches('-').to_string() } -/// Atomic + fsync'd file write: write a sibling temp file, fsync it, then rename -/// over the target (mirroring the session layer's durability). On any error the -/// temp file is removed so a crash mid-write can never leave a truncated memory -/// file. Memories are durable learnings, so they get the same crash-safety as -/// session persistence. +/// Atomic + fsync'd file write via a UNIQUE temp file (fsutil), so two +/// processes writing the same memory concurrently never collide on a shared +/// temp name and corrupt each other's write. Memories are durable learnings, +/// so they get the same crash-safety as session persistence (temp + fsync + +/// rename; an orphaned temp on crash is benign). fn atomic_write(path: &Path, content: &str) -> std::io::Result<()> { - let tmp = path.with_file_name(format!( - "{}.tmp", - path.file_name() - .map(|n| n.to_string_lossy().into_owned()) - .unwrap_or_default() - )); - { - let mut f = std::fs::File::create(&tmp)?; - f.write_all(content.as_bytes())?; - f.flush()?; - f.sync_all()?; - } - std::fs::rename(&tmp, path).inspect_err(|_e| { - let _ = std::fs::remove_file(&tmp); - }) + crate::fsutil::atomic_write_str(path, content) } // ---- tests ---- diff --git a/core/src/oauth.rs b/core/src/oauth.rs index b2db9ce..663f474 100644 --- a/core/src/oauth.rs +++ b/core/src/oauth.rs @@ -372,20 +372,9 @@ fn write_gemini_token(tok: &OAuthToken) -> Option<()> { .and_then(|s| serde_json::from_str::(&s).ok()); let creds = merged_gemini_creds(tok, existing.as_ref()); let data = serde_json::to_string_pretty(&creds).ok()?; - let tmp = path.with_extension("json.tmp"); - std::fs::write(&tmp, data).ok()?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let _ = std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o600)); - } - std::fs::rename(&tmp, &path).ok()?; - // chmod the final file too (rename preserves the tmp perms, but be safe). - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let _ = std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)); - } + // Unique-temp atomic write + 0600 (fsutil): two processes refreshing the + // same token concurrently never collide on a shared temp file. + crate::fsutil::atomic_write_secure(&path, data.as_bytes()).ok()?; Some(()) } @@ -425,14 +414,7 @@ fn store_token(provider: &str, tok: &OAuthToken) -> Option<()> { std::fs::create_dir_all(&dir).ok()?; let path = stored_token_path(provider)?; let data = serde_json::to_string_pretty(tok).ok()?; - let tmp = path.with_extension("json.tmp"); - std::fs::write(&tmp, data).ok()?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let _ = std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o600)); - } - std::fs::rename(&tmp, &path).ok()?; + crate::fsutil::atomic_write_secure(&path, data.as_bytes()).ok()?; Some(()) } @@ -509,14 +491,8 @@ fn write_codex_auth(tok: &OAuthToken, id_token: Option<&str>) -> Option<()> { "account_id": account_id, }, }); - let tmp = path.with_extension("json.tmp"); - std::fs::write(&tmp, serde_json::to_string_pretty(&data).ok()?).ok()?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let _ = std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o600)); - } - std::fs::rename(&tmp, &path).ok()?; + let serialized = serde_json::to_string_pretty(&data).ok()?; + crate::fsutil::atomic_write_secure(&path, serialized.as_bytes()).ok()?; Some(()) } diff --git a/core/src/pattern_log.rs b/core/src/pattern_log.rs index c478c57..5fc102e 100644 --- a/core/src/pattern_log.rs +++ b/core/src/pattern_log.rs @@ -72,6 +72,11 @@ impl Store { } let _ = std::fs::create_dir_all(&self.root); let path = self.path(workspace); + // Cross-process lock: append is a read-modify-write (read all lines, + // push, trim, write back). Two processes in the same workspace both + // completing a turn would otherwise race and silently drop entries. + // Advisory flock (auto-released on exit/crash). + let _lock = crate::fsutil::FileLock::acquire(&path.with_extension("lock")); let mut lines = read_lines(&path); let entry = PatternEntry { sig: sig.to_string(), @@ -143,10 +148,9 @@ fn read_entries(path: &Path) -> Vec { } fn write_atomic(path: &Path, content: &str) -> std::io::Result<()> { - let tmp = path.with_extension("jsonl.tmp"); - std::fs::write(&tmp, content)?; - std::fs::rename(&tmp, path)?; - Ok(()) + // Unique-temp atomic write (fsutil): two processes in the same workspace + // never share a temp file, so a concurrent append can't corrupt this one. + crate::fsutil::atomic_write_str(path, content) } /// Build a shape signature from the tool names used (in order) and the file diff --git a/core/src/plugins.rs b/core/src/plugins.rs index 4b2b11b..cb8dc02 100644 --- a/core/src/plugins.rs +++ b/core/src/plugins.rs @@ -100,8 +100,19 @@ JSON object to stdout before exiting. Stderr is captured for error reporting. the fields you want to change; everything else is preserved. Examples: pre_write `{ "content": "reformatted" }` overrides content but keeps `path`/ `edits`; pre_bash `{ "command": "fixed command" }` overrides the command; - pre_read `{ "path": "new/path" }` redirects the read. For post hooks, modify - is ignored (the operation already completed). + pre_read `{ "path": "new/path" }` redirects the read. For **post hooks**, + `modify` transforms the tool's RESULT: `{ "output": "...", "ok": false, + "diff": "..." }` replaces the result text, flips success, or replaces/clears + the diff — e.g. redact a secret, append context, or reformat. (The post + context includes the current result under the `result` key so the hook can + read it.) + + Note: pre-hook `modify` runs AFTER the approval gate + diff preview (which use + the original args), so a rewritten `path`/`command` is NOT re-prompted. File + tools still re-confine the path internally and `bash` re-checks its denylist, + so the security boundaries hold — but a plugin that redirects a safe path to a + sensitive one bypasses the user-facing prompt. Pre-hooks are trusted, + user-installed code (project hooks gated by `--trust-project-plugins`). Safety rules enforced by the core: - pre_* hooks: non-zero exit, timeout, or JSON parse failure → `allow: false` (blocks the tool) @@ -120,6 +131,8 @@ Safety rules enforced by the core: | post_bash | After a bash command completes | post | | post_write | After a file write/edit completes | post | | post_read | After a file is read | post | +| pre_tool | Before ANY tool executes (catch-all) | pre | +| post_tool | After ANY tool executes (catch-all) | post | | session_start | When a session begins (prompt received) | lifecycle | | session_stop | When a session ends (done/abort) | lifecycle | | pre_compact | Before conversation compaction | pre | @@ -182,9 +195,10 @@ hooks, or both): ``` Fields: -- `name` (required): tool name. Must not collide with a built-in tool - (bash, read_file, edit, subagent, …) — built-ins always win, so a colliding - plugin tool is skipped with a warning. +- `name` (required): tool name. By default it must not collide with a built-in + tool (bash, read_file, edit, subagent, …) — a colliding plugin tool is skipped + and the built-in wins. Set `override: true` (below) to instead REPLACE the + built-in's implementation with this plugin's handler. - `description` (optional): shown to the model. - `parameters` (optional): a JSON Schema for the tool's arguments. Defaults to an empty object. @@ -193,6 +207,12 @@ Fields: - `kind` (optional): `"readonly"` (skips the approval gate) or `"destructive"` (prompts under Approval::Destructive — the default). Arbitrary external code runs on every call, so default to `destructive`. +- `override` (optional, bool): when `true` AND `name` matches a built-in tool, + this plugin's handler REPLACES that built-in — the model still sees a tool of + that name (this plugin's declared `description`/`parameters`), but calls route + to the plugin script instead of the core handler. This is the no-recompile way + to fully override a core tool (a sandboxed `bash`, a redacting `read_file`, a + rate-limited `git_commit`, …). Default `false`: a name collision stays built-in. - `timeout_ms` (optional): hard per-call timeout (default 30s). Tool handler contract (one JSON object on stdin, one on stdout): @@ -232,6 +252,75 @@ jq -n --arg o "Order #$id: shipped" '{ "ok": true, "output": $o }' Remember: `chmod +x` the handler. +### Add, override, and remove core behavior + +A plugin can do everything a direct core edit can — add, override, and remove +behavior — without recompiling. Five mechanisms cover the full surface: + +| Operation | Mechanism | Notes | +|-----------|-----------|-------| +| **ADD a tool** | `tools` array | A new capability the model can call. | +| **OVERRIDE a tool** | a tool with `override: true` | Replaces a built-in's implementation (the model still sees that tool name; calls route to the plugin script). | +| **REMOVE a tool** | `disable_tools` | Drops the named tool from the model's toolset entirely (built-in or override). The strongest lever — wins over `override`. | +| **MODIFY tool input** | `pre_bash`/`pre_write`/`pre_read`/`pre_tool` `modify` | Override specific args before execution. | +| **MODIFY tool output** | `post_*`/`post_tool` `modify` | Replace the result text / flip success / change the diff after execution. | +| **MODIFY the model** | `pre_turn` `modify.model` | Remap the turn's model (advisory). | +| **ADD to the system prompt** | `system_prompt` field | Static text appended to the system prompt. | + +#### `disable_tools` — remove a capability + +```json +{ + "name": "no-bash", + "version": "1.0.0", + "disable_tools": ["bash", "git_commit"] +} +``` + +The listed tool names vanish from the model's toolset — it can never call them +(this is stronger than a per-call `pre_bash` deny). Composes across plugins (the +union is removed). Applied as a final filter, so it also removes a tool another +plugin `override`s. + +#### `system_prompt` — inject context + +```json +{ + "name": "domain-rules", + "version": "1.0.0", + "system_prompt": "All database access must go through the `db_query` tool. Never construct raw SQL in bash." +} +``` + +The text is appended to the system prompt (after the plugin docs), framed with +the plugin name + version — the same surface a core edit of the system prompt +touches. Empty by default, so the prompt + its prefix cache are untouched when no +plugin declares one. (Main agent only; subagents use the built-in tool set.) + +#### `override: true` — replace a core tool + +```json +{ + "name": "sandboxed-bash", + "version": "1.0.0", + "tools": [ + { + "name": "bash", + "override": true, + "description": "Run a command in the project sandbox.", + "parameters": {"type":"object","properties":{"command":{"type":"string"}},"required":["command"]}, + "script": "tools/bash.sh", + "kind": "destructive" + } + ] +} +``` + +The model calls `bash` as usual, but the call routes to `tools/bash.sh` instead +of the core handler — and the plugin controls the description/schema. The tool's +`kind` (approval gate) is the plugin's. The specific `pre_bash`/`post_bash` +hooks still fire (keyed on the tool name), and `pre_tool`/`post_tool` fire too. + ### Example: a pre_write linter plugin `.catalyst-code/plugins/lint-check/plugin.json`: @@ -282,6 +371,14 @@ pub const HOOK_POINTS: &[&str] = &[ "session_stop", "pre_compact", "pre_turn", + // Catch-all hooks that fire for EVERY tool call (in addition to the + // specific pre_bash/pre_write/pre_read). They cover tools with no + // dedicated hook (memory, todo_write, git_*, subagent, plugin tools, …) + // so a plugin can audit/modify/deny ANY tool — the same reach a core edit + // of the dispatch loop has. pre_tool runs after the specific pre-hook; + // post_tool runs after the specific post-hook. + "pre_tool", + "post_tool", ]; /// Default timeout in milliseconds for pre_* hooks (blocking — keep short). @@ -303,6 +400,12 @@ struct PluginManifest { /// Optional user-declared tools (custom capabilities, no MCP needed). #[serde(default)] tools: Vec, + /// Built-in/plugin tool names to REMOVE from the model's toolset. + #[serde(default)] + disable_tools: Vec, + /// Static text injected into the system prompt (empty = none). + #[serde(default)] + system_prompt: String, } #[derive(Deserialize, Debug, Clone)] @@ -330,6 +433,14 @@ struct ToolManifestEntry { kind: Option, #[serde(default)] timeout_ms: Option, + /// When true AND `name` matches a built-in tool, this plugin's handler + /// REPLACES the built-in's implementation: the model still sees a tool of + /// that name (the plugin's declared schema), but calls route to the plugin + /// script instead of the core handler. Lets a plugin fully override a + /// core tool (a sandboxed bash, a redacting read_file, …) without + /// recompiling. Default false: a name collision stays built-in (unchanged). + #[serde(default, rename = "override")] + override_builtin: bool, } // ---- public types ---- @@ -347,6 +458,10 @@ pub struct Plugin { pub hooks: HashMap, /// Tools this plugin declares (custom capabilities; no MCP needed). pub tools: Vec, + /// Built-in/plugin tool names to REMOVE from the model's toolset. + pub disable_tools: Vec, + /// Static text injected into the system prompt (empty = none). + pub system_prompt: String, } /// Configuration for one hook within a plugin. @@ -373,6 +488,8 @@ pub struct ToolConfig { pub timeout_ms: u64, /// Approval classification: ReadOnly skips the gate, Destructive prompts. pub kind: ToolKind, + /// True → this tool's handler replaces the built-in of the same name. + pub override_builtin: bool, } /// Result returned from executing a hook. @@ -710,6 +827,7 @@ impl PluginManager { script: canon_script, timeout_ms, kind, + override_builtin: t.override_builtin, }); } @@ -721,6 +839,8 @@ impl PluginManager { source_path: canon_dir, hooks, tools: tools_vec, + disable_tools: manifest.disable_tools, + system_prompt: manifest.system_prompt, }) } @@ -828,6 +948,18 @@ impl PluginManager { .collect() } + /// Cheap existence check (no config clone): does any enabled plugin register + /// this hook point? Used to decide whether to clone tool args before the + /// pre-hook phase without paying for a full `get_hook_configs`. + pub fn has_hook(&self, hook_name: &str) -> bool { + self.plugins + .read() + .unwrap() + .values() + .filter(|p| p.enabled) + .any(|p| p.hooks.contains_key(hook_name)) + } + /// Look up a single plugin by name. pub fn get_plugin(&self, name: &str) -> Option { self.plugins.read().unwrap().get(name).cloned() @@ -869,6 +1001,59 @@ impl PluginManager { pub fn tool_kind(&self, name: &str) -> Option { self.tool_config(name).map(|t| t.kind) } + + /// Union of tool names every enabled plugin asks to disable (the + /// `disable_tools` manifest field). Applied as a FINAL filter on the + /// model's tool list, so a disabled name is gone whether it's a built-in + /// or an override — `disable_tools` is the strongest "remove a feature" + /// lever and always wins over `override`. + pub fn disabled_tools(&self) -> std::collections::HashSet { + self.plugins + .read() + .unwrap() + .values() + .filter(|p| p.enabled) + .flat_map(|p| p.disable_tools.iter().cloned()) + .collect() + } + + /// Built-in tool names for which an enabled plugin declares an + /// `override: true` tool — the plugin's handler replaces the built-in's + /// implementation. A plugin tool named like a built-in WITHOUT + /// `override: true` does NOT appear here (it stays a no-op collision, + /// built-in wins — unchanged behavior). + pub fn overridden_tool_names(&self) -> std::collections::HashSet { + self.plugins + .read() + .unwrap() + .values() + .filter(|p| p.enabled) + .flat_map(|p| p.tools.iter()) + .filter(|t| t.override_builtin && crate::tools::is_builtin(&t.name)) + .map(|t| t.name.clone()) + .collect() + } + + /// Concatenated `system_prompt` text from every enabled plugin that + /// declares one, each framed with its plugin name + version. Empty (so the + /// system prompt + its prefix cache are untouched) when no plugin declares + /// any. Lets a plugin inject domain rules / persona / context into the + /// system prompt — the same surface a core edit of SYSTEM_PROMPT_BASE + /// touches. + pub fn system_prompt_injection(&self) -> String { + let mut parts: Vec = Vec::new(); + for p in self.plugins.read().unwrap().values().filter(|p| p.enabled) { + let s = p.system_prompt.trim(); + if !s.is_empty() { + parts.push(format!("# Plugin: {} (v{})\n{}", p.name, p.version, s)); + } + } + if parts.is_empty() { + String::new() + } else { + format!("\n\n## Plugin-injected context\n\n{}", parts.join("\n\n")) + } + } } // ---- hook execution ---- @@ -2272,6 +2457,178 @@ mod tests { // ---- execute_plugin_tool ---- + /// Write a `plugin.json` with arbitrary manifest JSON into `dir`. + fn write_manifest(dir: &Path, manifest: &str) { + fs::write(dir.join("plugin.json"), manifest).unwrap(); + } + + #[test] + fn hook_points_include_catch_all() { + assert!(HOOK_POINTS.contains(&"pre_tool")); + assert!(HOOK_POINTS.contains(&"post_tool")); + // pre_* get the short timeout; post_* the long one. + assert_eq!(default_hook_timeout("pre_tool"), DEFAULT_PRE_TIMEOUT_MS); + assert_eq!(default_hook_timeout("post_tool"), DEFAULT_POST_TIMEOUT_MS); + } + + #[test] + fn disable_tools_manifest_loaded() { + let tmp = TmpDir::new("disable_loaded"); + write_manifest( + &tmp.path, + r#"{"name":"no-bash","version":"1.0.0","disable_tools":["bash","git_commit"]}"#, + ); + let plugin = PluginManager::load_plugin_from_dir(&tmp.path).unwrap(); + assert_eq!( + plugin.disable_tools, + vec!["bash".to_string(), "git_commit".to_string()] + ); + } + + #[test] + fn system_prompt_manifest_loaded() { + let tmp = TmpDir::new("sysprompt_loaded"); + write_manifest( + &tmp.path, + r#"{"name":"rules","version":"2.0.0","system_prompt":"Never run raw SQL."}"#, + ); + let plugin = PluginManager::load_plugin_from_dir(&tmp.path).unwrap(); + assert_eq!(plugin.system_prompt, "Never run raw SQL."); + } + + #[test] + fn override_field_loaded() { + let tmp = TmpDir::new("override_loaded"); + let pdir = write_plugin_with_tool(&tmp.path, "bash", r#""override":true"#); + let plugin = PluginManager::load_plugin_from_dir(&pdir).unwrap(); + assert!(plugin.tools[0].override_builtin); + + // Without override, it stays false. + let tmp2 = TmpDir::new("override_false"); + let pdir2 = write_plugin_with_tool(&tmp2.path, "my_tool", ""); + let plugin2 = PluginManager::load_plugin_from_dir(&pdir2).unwrap(); + assert!(!plugin2.tools[0].override_builtin); + } + + #[test] + fn manager_disabled_tools_unions_across_plugins() { + let tmp = TmpDir::new("disable_union"); + let a = tmp.path.join("a"); + let b = tmp.path.join("b"); + fs::create_dir_all(&a).unwrap(); + fs::create_dir_all(&b).unwrap(); + write_manifest( + &a, + r#"{"name":"a","version":"1.0.0","disable_tools":["bash"]}"#, + ); + write_manifest( + &b, + r#"{"name":"b","version":"1.0.0","disable_tools":["bash","edit"]}"#, + ); + let mgr = PluginManager::new(tmp.path.clone(), PathBuf::from("/__t_ws__"), true); + let disabled = mgr.disabled_tools(); + assert!(disabled.contains("bash")); + assert!(disabled.contains("edit")); + assert_eq!(disabled.len(), 2); + } + + #[test] + fn overridden_tool_names_only_when_override_and_builtin() { + // override:true on a built-in name → overridden. override:false (or a + // custom name) → NOT overridden. + let tmp = TmpDir::new("override_names"); + write_plugin_with_tool(&tmp.path, "bash", r#""override":true"#); + let mgr = PluginManager::new(tmp.path.clone(), PathBuf::from("/__t_ws__"), true); + let names = mgr.overridden_tool_names(); + assert!(names.contains("bash")); + + // override:true on a NON-built-in name → not in overridden set (there's + // nothing to override; it's just a custom tool). + let tmp2 = TmpDir::new("override_custom"); + write_plugin_with_tool(&tmp2.path, "my_domain_tool", r#""override":true"#); + let mgr2 = PluginManager::new(tmp2.path.clone(), PathBuf::from("/__t_ws__"), true); + assert!(mgr2.overridden_tool_names().is_empty()); + + // A plain collision (no override) on a built-in → NOT overridden. + let tmp3 = TmpDir::new("override_none"); + write_plugin_with_tool(&tmp3.path, "read_file", r#""kind":"readonly""#); + let mgr3 = PluginManager::new(tmp3.path.clone(), PathBuf::from("/__t_ws__"), true); + assert!(mgr3.overridden_tool_names().is_empty()); + } + + #[test] + fn system_prompt_injection_concat_and_framed() { + let tmp = TmpDir::new("sysprompt_inject"); + let a = tmp.path.join("a"); + let b = tmp.path.join("b"); + fs::create_dir_all(&a).unwrap(); + fs::create_dir_all(&b).unwrap(); + write_manifest( + &a, + r#"{"name":"alpha","version":"1.0.0","system_prompt":"rule A"}"#, + ); + write_manifest( + &b, + r#"{"name":"beta","version":"2.0.0","system_prompt":"rule B"}"#, + ); + let mgr = PluginManager::new(tmp.path.clone(), PathBuf::from("/__t_ws__"), true); + let inj = mgr.system_prompt_injection(); + assert!(inj.starts_with("\n\n## Plugin-injected context\n\n")); + assert!(inj.contains("# Plugin: alpha (v1.0.0)\nrule A")); + assert!(inj.contains("# Plugin: beta (v2.0.0)\nrule B")); + + // Empty when no plugin declares one (prefix-cache-safe). + let tmp2 = TmpDir::new("sysprompt_empty"); + write_manifest(&tmp2.path, r#"{"name":"plain","version":"1.0.0"}"#); + let mgr2 = PluginManager::new(tmp2.path.clone(), PathBuf::from("/__t_ws__"), true); + assert!(mgr2.system_prompt_injection().is_empty()); + } + + #[test] + fn has_hook_existence_check() { + let tmp = TmpDir::new("has_hook"); + // PluginManager::new scans SUBDIRECTORIES of its root for plugins, + // so the plugin must live in a subdir. + let pdir = tmp.path.join("h"); + let hooks_dir = pdir.join("hooks"); + fs::create_dir_all(&hooks_dir).unwrap(); + write_hook_script(&hooks_dir, "h.sh", r#"{"allow":true}"#, 0); + write_manifest( + &pdir, + r#"{"name":"h","version":"1.0.0","hooks":{"pre_tool":{"script":"hooks/h.sh"}}}"#, + ); + let mgr = PluginManager::new(tmp.path.clone(), PathBuf::from("/__t_ws__"), true); + assert!(mgr.has_hook("pre_tool")); + assert!(!mgr.has_hook("pre_bash")); + // Disabled plugin is excluded. + mgr.disable("h").unwrap(); + assert!(!mgr.has_hook("pre_tool")); + } + + #[test] + fn disabled_plugin_excluded_from_new_capabilities() { + // A disabled plugin contributes nothing to disable_tools / overrides / + // system_prompt — mirroring how disabled plugins are excluded from + // hook configs and tool definitions. + let tmp = TmpDir::new("disabled_excluded"); + let pdir = write_plugin_with_tool(&tmp.path, "bash", r#""override":true"#); + // Augment with disable_tools + system_prompt. + let manifest = fs::read_to_string(pdir.join("plugin.json")).unwrap(); + let manifest = manifest.trim_end_matches('}').to_string() + + r#","disable_tools":["edit"],"system_prompt":"ctx"}"#; + fs::write(pdir.join("plugin.json"), &manifest).unwrap(); + + let mgr = PluginManager::new(tmp.path.clone(), PathBuf::from("/__t_ws__"), true); + assert!(mgr.overridden_tool_names().contains("bash")); + assert!(mgr.disabled_tools().contains("edit")); + assert!(mgr.system_prompt_injection().contains("ctx")); + + mgr.disable("tools-plugin").unwrap(); + assert!(mgr.overridden_tool_names().is_empty()); + assert!(mgr.disabled_tools().is_empty()); + assert!(mgr.system_prompt_injection().is_empty()); + } + fn tool_config_for(script: PathBuf, timeout_ms: u64, kind: ToolKind) -> ToolConfig { ToolConfig { name: "ut".into(), @@ -2280,6 +2637,7 @@ mod tests { script, timeout_ms, kind, + override_builtin: false, } } diff --git a/core/src/presence.rs b/core/src/presence.rs new file mode 100644 index 0000000..6197c7f --- /dev/null +++ b/core/src/presence.rs @@ -0,0 +1,254 @@ +// Cross-session workspace presence: each core process publishes a small +// "I'm here and doing X" record so other sessions in the SAME workspace can +// detect concurrent activity — instead of blaming themselves for phantom errors +// caused by a neighbor's in-flight edits. +// +// Per-pid JSON files under ~/.config/catalyst-code/presence//.json. +// Per-pid (not one shared file) → zero write contention; each process owns one +// file. Writes are atomic (temp + fsync + rename) — the same crash-safety +// pattern as session/memory persistence. Stale records (a crashed/killed +// process that stopped rewriting its file) are reaped by mtime on read, so a +// `kill -9` is tolerated: the next reader deletes the dead file. +// +// This is AWARENESS ONLY — read-only broadcast of "who is here and what are +// they touching." It deliberately does NOT coordinate (no locking, no +// work-claiming): partial coordination is more dangerous than none, and 80% of +// the value is an agent *knowing* a neighbor is active so it stops "fixing" +// phantom errors and corrupting in-flight work. See the `workspace_activity` +// tool and the `maybe_concurrency_note` anomaly nudge in main.rs. +use crate::config::home_dir; +use crate::memory::project_hash; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; + +/// A record older than this (by file mtime) is considered stale/dead and is +/// reaped on read. The heartbeat rewrites the file every ~8s, so a live process +/// is never within 30s of this threshold — `kill -9` / a crashed core leaves a +/// file the next reader deletes. +const STALE_SECS: u64 = 30; + +/// One session's published presence. Serialized pretty-printed to disk so a +/// human can eyeball `.json`; small files. +#[derive(Clone, Serialize, Deserialize)] +pub struct PresenceRecord { + pub pid: u32, + pub session_id: Option, + /// Unix seconds — when this session started. For "started Xm ago". + pub started_at: u64, + /// Unix seconds — rewritten every heartbeat. For human readability; the + /// reap decision uses file mtime (the reliable signal), not this field. + pub last_heartbeat: u64, + pub goal: String, + pub in_progress: Vec, + pub next: Vec, + pub recent_files: Vec, + pub last_activity: String, + pub model: Option, +} + +impl PresenceRecord { + /// Build from the session's rolling work-state + identifying context. + pub fn from_work_state( + ws: &crate::WorkState, + pid: u32, + session_id: Option, + model: Option, + started_at: u64, + ) -> Self { + Self { + pid, + session_id, + started_at, + last_heartbeat: unix_now(), + goal: ws.goal.clone(), + in_progress: ws.in_progress.clone(), + next: ws.next.clone(), + recent_files: ws.recent_files.clone(), + last_activity: ws.last_activity.clone(), + model, + } + } +} + +/// The per-workspace presence directory: ~/.config/catalyst-code/presence//. +/// Returns None if the home dir can't be determined (presence disabled). +pub fn presence_dir(workspace: &Path) -> Option { + let home = home_dir()?; + Some( + home.join(".config/catalyst-code/presence") + .join(project_hash(&workspace.to_string_lossy())), + ) +} + +/// The per-process presence file: /.json. +pub fn presence_file(workspace: &Path, pid: u32) -> Option { + Some(presence_dir(workspace)?.join(format!("{pid}.json"))) +} + +/// Atomically write (or overwrite) our presence record. Best-effort — presence +/// is advisory, so a write failure is logged to stderr, never fatal. +pub fn write_presence(workspace: &Path, pid: u32, rec: &PresenceRecord) { + let Some(file) = presence_file(workspace, pid) else { + return; + }; + if let Err(e) = atomic_write_json(&file, rec) { + eprintln!("[presence] failed to write {}: {e}", file.display()); + } +} + +/// Delete our presence file on clean shutdown. Best-effort; stale-reaping on +/// read is the real correctness net (covers `kill -9` / crash). +pub fn clear_presence(workspace: &Path, pid: u32) { + if let Some(file) = presence_file(workspace, pid) { + let _ = std::fs::remove_file(file); + } +} + +/// Read all LIVE peer records for this workspace (excluding our own pid). +/// Stale records (mtime older than STALE_SECS) are reaped (deleted) and +/// skipped. Unparseable files are skipped (mtime will reap them later). +pub fn read_peers(workspace: &Path, my_pid: u32) -> Vec { + let Some(dir) = presence_dir(workspace) else { + return Vec::new(); + }; + let entries = match std::fs::read_dir(&dir) { + Ok(e) => e, + Err(_) => return Vec::new(), // dir doesn't exist yet → no peers + }; + let now = unix_now(); + let mut peers = Vec::new(); + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) != Some("json") { + continue; + } + // Reap by mtime: a process that hasn't rewritten its file in STALE_SECS + // is dead. mtime is the reliable signal (the heartbeat updates it every + // ~8s); last_heartbeat is for human readability only. + let mtime_secs = match entry + .metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + { + Some(d) => d.as_secs(), + None => continue, + }; + if now.saturating_sub(mtime_secs) > STALE_SECS { + let _ = std::fs::remove_file(&path); // reap stale + continue; + } + let rec = match std::fs::read_to_string(&path) + .ok() + .and_then(|s| serde_json::from_str::(&s).ok()) + { + Some(r) => r, + None => continue, // unparseable — leave it; mtime will reap + }; + if rec.pid == my_pid { + continue; // skip self + } + peers.push(rec); + } + peers +} + +/// Current unix timestamp in seconds (0 on clock error). +pub fn unix_now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +/// Atomic write: temp file in the same dir + fsync + rename. Same crash-safety +/// pattern as session/memory persistence. A crash mid-write leaves the temp +/// (hidden, `..tmp`) orphaned, never a truncated record. +fn atomic_write_json(path: &Path, rec: &PresenceRecord) -> std::io::Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let fname = path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("presence.json"); + let tmp = path.with_file_name(format!(".{fname}.tmp")); + { + use std::io::Write; + let mut f = std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(&tmp)?; + let body = serde_json::to_vec_pretty(rec).map_err(std::io::Error::other)?; + f.write_all(&body)?; + f.sync_all()?; + } + std::fs::rename(&tmp, path)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::WorkState; + + fn ws_dir() -> std::path::PathBuf { + // A unique temp workspace so parallel tests don't collide. + let dir = std::env::temp_dir().join(format!( + "catalyst-presence-test-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn write_then_read_peer() { + let dir = ws_dir(); + let ws = WorkState { + goal: "refactor auth".into(), + in_progress: vec!["core/src/main.rs".into()], + ..Default::default() + }; + let rec = + PresenceRecord::from_work_state(&ws, 4242, Some("s.json".into()), None, unix_now()); + write_presence(&dir, 4242, &rec); + + // Self is excluded. + assert!(read_peers(&dir, 4242).is_empty()); + + // Another pid sees it. + let peers = read_peers(&dir, 9999); + assert_eq!(peers.len(), 1); + assert_eq!(peers[0].pid, 4242); + assert_eq!(peers[0].goal, "refactor auth"); + assert_eq!(peers[0].recent_files, Vec::::new()); + assert_eq!(peers[0].in_progress, vec!["core/src/main.rs".to_string()]); + + // Cleanup clears our own file but leaves peers. + clear_presence(&dir, 4242); + assert!(read_peers(&dir, 9999).is_empty()); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn stale_record_is_reaped() { + let dir = ws_dir(); + // Write a record, then backdate its mtime beyond STALE_SECS. + let rec = + PresenceRecord::from_work_state(&WorkState::default(), 5555, None, None, unix_now()); + write_presence(&dir, 5555, &rec); + let file = presence_file(&dir, 5555).unwrap(); + let past = std::time::SystemTime::now() - std::time::Duration::from_secs(STALE_SECS + 60); + let _ = filetime::set_file_mtime(&file, filetime::FileTime::from_system_time(past)); + + // Stale → reaped (file deleted), not returned. + assert!(read_peers(&dir, 9999).is_empty()); + assert!(!file.exists(), "stale presence file should be reaped"); + std::fs::remove_dir_all(&dir).ok(); + } +} diff --git a/core/src/protocol.rs b/core/src/protocol.rs index fbcab5c..34cf050 100644 --- a/core/src/protocol.rs +++ b/core/src/protocol.rs @@ -125,9 +125,15 @@ pub enum Command { /// Drop the last turn (user prompt + its assistant reply + tool calls/results). #[serde(rename = "undo")] Undo, - /// Force a context compaction now (regardless of the 70% threshold). + /// Force a context compaction now (regardless of the threshold). Optional + /// `instructions` override `compact_instructions` for this call only (e.g. + /// `/compact Focus on code samples and API usage`); empty/absent falls back + /// to the configured default. Always works regardless of `auto_compact`. #[serde(rename = "compact")] - Compact, + Compact { + #[serde(default)] + instructions: Option, + }, /// List available session files (returns a `sessions` event). #[serde(rename = "list_sessions")] ListSessions, @@ -145,6 +151,11 @@ pub enum Command { /// Request a stats summary (returns a `stats` event). #[serde(rename = "stats")] Stats, + /// Request a token-usage breakdown of the current context (returns a + /// `context_breakdown` event): total/context-window/pct, per-role buckets, + /// and the top token consumers (biggest messages). Read-only. + #[serde(rename = "context")] + Context, /// Approve a pending tool call. decision: "yes" | "no" | "always". /// "always" upgrades the session approval mode so subsequent same-tool calls skip the gate. #[serde(rename = "approve")] diff --git a/core/src/provider.rs b/core/src/provider.rs index 67a4a41..c224fc5 100644 --- a/core/src/provider.rs +++ b/core/src/provider.rs @@ -167,14 +167,21 @@ pub async fn summarize( model: &str, messages: &[Message], cancel: &CancellationToken, + instructions: Option<&str>, ) -> Option { - const SYS: &str = "Summarize the following conversation turns in structured format. Preserve: decisions made, file paths touched, the user's goal, and any unresolved errors.\n\nUse this exact format:\n\n 1. Primary Request and Intent\n 2. Key Technical Concepts\n 3. Files and Code Sections\n 4. Errors and Fixes\n 5. Problem Solving\n 6. All User Messages\n 7. Pending Tasks\n 8. Current Work\n 9. Optional Next Step\n"; + const BASE_SYS: &str = "Summarize the following conversation turns in structured format. Preserve: decisions made, file paths touched, the user's goal, and any unresolved errors.\n\nUse this exact format:\n\n 1. Primary Request and Intent\n 2. Key Technical Concepts\n 3. Files and Code Sections\n 4. Errors and Fixes\n 5. Problem Solving\n 6. All User Messages\n 7. Pending Tasks\n 8. Current Work\n 9. Optional Next Step\n"; + let sys = match instructions.map(str::trim).filter(|s| !s.is_empty()) { + Some(extra) => format!( + "{BASE_SYS}\n\nThe user provided the following guidance for what to preserve in this summary — honor it above the default priorities:\n{extra}" + ), + None => BASE_SYS.to_string(), + }; let user = messages .iter() .map(message_for_summary) .collect::>() .join("\n"); - complete_text(client, provider, model, SYS, &user, 1024, cancel).await + complete_text(client, provider, model, &sys, &user, 1024, cancel).await } /// Extract durable facts worth remembering across future sessions from a slice of @@ -595,6 +602,15 @@ fn write_models_cache(cache_key: &str, models: &[ModelInfo]) { // multi-provider caches then all hit on the next startup instead of only // the last writer's. Written atomically (temp + fsync + rename) so a crash // mid-write can't truncate/corrupt the cache file. + // Cross-process lock: the cache is a shared read-modify-write (we merge + // this provider's entry into the existing entries map). Without a lock two + // processes refreshing different providers concurrently would both read the + // same base and the second rename would clobber the first's entry. Advisory + // (flock); auto-releases on exit/crash so there are no stale locks. + let _lock = match crate::fsutil::FileLock::acquire(&path.with_extension("lock")) { + Ok(g) => g, + Err(_) => return, // best-effort: never block the turn on a wedged lock + }; let mut entries: serde_json::Map = std::fs::read_to_string(&path) .ok() .and_then(|c| serde_json::from_str::(&c).ok()) @@ -610,35 +626,14 @@ fn write_models_cache(cache_key: &str, models: &[ModelInfo]) { "version": MODELS_CACHE_VERSION, "entries": entries, }); - atomic_write_cache_file( + // Unique-temp atomic write (fsutil): two processes never share a temp file, + // so a concurrent writer can't corrupt this one's write. + let _ = crate::fsutil::atomic_write_str( &path, &serde_json::to_string(&cache).unwrap_or_else(|_| "{}".into()), ); } -/// Atomically write `content` to the models-cache `path`: temp file + fsync + -/// rename, so a crash/SIGKILL/OOM mid-write can never leave the cache truncated -/// or partially written (bare `std::fs::write` is truncate-then-write). -fn atomic_write_cache_file(path: &std::path::Path, content: &str) { - use std::io::Write; - let tmp: std::path::PathBuf = { - let mut p = path.as_os_str().to_owned(); - p.push(".tmp"); - std::path::PathBuf::from(p) - }; - let res = (|| -> std::io::Result<()> { - let mut f = std::fs::File::create(&tmp)?; - f.write_all(content.as_bytes())?; - f.sync_all()?; - drop(f); - std::fs::rename(&tmp, path)?; - Ok(()) - })(); - if res.is_err() { - let _ = std::fs::remove_file(&tmp); - } -} - fn parse_cache_models(cache: &Value) -> Option> { let arr = cache.get("models")?.as_array()?; let mut out = Vec::new(); @@ -4726,7 +4721,7 @@ mod tests { Message::user("please refactor the auth module"), Message::assistant("on it"), ]; - let out = summarize(&client, &provider, "mock-model", &msgs, &cancel).await; + let out = summarize(&client, &provider, "mock-model", &msgs, &cancel, None).await; assert_eq!(out.as_deref(), Some("mocked")); } @@ -4753,7 +4748,7 @@ mod tests { let provider = mock_provider(base); let cancel = CancellationToken::new(); let msgs: Vec = vec![Message::user("x")]; - let out = summarize(&client, &provider, "mock-model", &msgs, &cancel).await; + let out = summarize(&client, &provider, "mock-model", &msgs, &cancel, None).await; assert!(out.is_none()); } } diff --git a/core/src/staging.rs b/core/src/staging.rs index a459f67..b980b6f 100644 --- a/core/src/staging.rs +++ b/core/src/staging.rs @@ -22,7 +22,6 @@ //! untouched. use crate::config::home_dir; -use std::io::Write; use std::path::PathBuf; /// Bump when the bundled default set changes meaningfully. The marker file @@ -237,16 +236,11 @@ fn executable_rel_paths() -> &'static [&'static str] { /// short-circuit would then treat as complete (and never re-stage). Keeps the /// idempotent/non-clobbering semantics — only the write durability changes. fn atomic_write(path: &std::path::Path, content: &str) -> std::io::Result<()> { - let tmp = path.with_extension("tmp"); - { - let mut f = std::fs::File::create(&tmp)?; - f.write_all(content.as_bytes())?; - f.flush()?; - f.sync_all()?; - } - std::fs::rename(&tmp, path).inspect_err(|_e| { - let _ = std::fs::remove_file(&tmp); - }) + // Unique-temp (fsutil): two processes staging on first run simultaneously + // never collide on a shared temp file. Staging is idempotent + + // non-clobbering (the exists() check skips present files), so no + // cross-process lock is needed here — just a non-corrupting write. + crate::fsutil::atomic_write_str(path, content) } /// Ensure the global default files exist under `~/.catalyst-code/`. Writes only diff --git a/core/src/subagent.rs b/core/src/subagent.rs index 7ecd586..a0c9c21 100644 --- a/core/src/subagent.rs +++ b/core/src/subagent.rs @@ -1311,6 +1311,7 @@ async fn run_agent_inner( cancel, est > hard_cap, model_ctx, + cfg.compact_instructions.as_deref(), ) .await; // Compaction rewrote `sub`; the real baseline no longer applies. diff --git a/core/src/tools.rs b/core/src/tools.rs index 744507b..56125aa 100644 --- a/core/src/tools.rs +++ b/core/src/tools.rs @@ -1,7 +1,7 @@ // Built-in tools the agent can call. OpenAI function-calling schema. // All file ops are confined to the workspace root; bash runs with cwd=workspace // and a real timeout+kill. read_file returns plain content; edit uses search/replace. -use crate::config::Config; +use crate::config::{Approval, Config}; use crate::workspace; use serde_json::{json, Value}; @@ -23,6 +23,7 @@ pub fn classify(name: &str) -> ToolKind { | "memory" => ToolKind::ReadOnly, "web_search" => ToolKind::ReadOnly, "ask" => ToolKind::ReadOnly, + "workspace_activity" => ToolKind::ReadOnly, _ => ToolKind::Destructive, } } @@ -465,6 +466,17 @@ pub fn definitions() -> Vec { } } }), + json!({ + "type": "function", + "function": { + "name": "workspace_activity", + "description": "List OTHER active catalyst-code agent sessions running in THIS workspace (separate processes), with each one's goal, what it's working on, and the files it recently touched. Use this when something seems off (a build failing for reasons you didn't cause, a file that changed unexpectedly, a test suddenly breaking) to check whether another session is the cause before assuming you introduced the error. Read-only — awareness only, no coordination. Returns the live peers (stale/crashed sessions are auto-pruned).", + "parameters": { + "type": "object", + "properties": {} + } + } + }), json!({ "type": "function", "function": { @@ -615,6 +627,7 @@ pub fn execute(name: &str, args: &Value, cfg: &Config) -> Outcome { "git_status" => git_status(args, cfg), "git_diff" => git_diff(args, cfg), "git_log" => git_log(args, cfg), + "workspace_activity" => workspace_activity(args, cfg), "git_add" => git_add(args, cfg), "git_commit" => git_commit(args, cfg), "memory" => memory_tool(args, cfg), @@ -643,8 +656,24 @@ impl Outcome { // ---- file tools ---- +/// Resolve a tool's path argument against the workspace root, honoring the +/// approval mode. Under `Approval::Never` ALL path confinement is disabled — +/// absolute paths, `..` traversal, and symlink escapes are allowed (the model +/// is fully trusted, so it may read/write anywhere on the host). Under +/// `Destructive`/`Always` the full confinement applies (reject absolute, +/// reject `..`, reject symlink-outside-workspace). The dangerous-path list +/// (.env/.git/.ssh) is gated separately in the approval gate +/// (main::restricted_path_for_tool), which is also Never-off. +fn resolve_ws(cfg: &Config, input: &str) -> Result { + if matches!(cfg.approval, Approval::Never) { + workspace::resolve_unconfined(&cfg.workspace, input) + } else { + workspace::resolve(&cfg.workspace, input) + } +} + fn read_file(input: &str, args: &Value, cfg: &Config) -> Outcome { - let path = match workspace::resolve(&cfg.workspace, input) { + let path = match resolve_ws(cfg, input) { Ok(p) => p, Err(e) => return Outcome::err(e), }; @@ -734,7 +763,7 @@ fn write_file(input: &str, content: &str, cfg: &Config) -> Outcome { // (main::restricted_path_for_tool) so that under Approval::Never ALL // restrictions are disabled, and under Destructive/Always a restricted // path prompts (instead of an unconditional kill) for reads AND writes. - let path = match workspace::resolve(&cfg.workspace, input) { + let path = match resolve_ws(cfg, input) { Ok(p) => p, Err(e) => return Outcome::err(e), }; @@ -757,7 +786,7 @@ fn write_file(input: &str, content: &str, cfg: &Config) -> Outcome { } fn list_dir(input: &str, cfg: &Config) -> Outcome { - let path = match workspace::resolve(&cfg.workspace, input) { + let path = match resolve_ws(cfg, input) { Ok(p) => p, Err(e) => return Outcome::err(e), }; @@ -790,7 +819,7 @@ fn grep(pattern: &str, input: &str, context: usize, cfg: &Config) -> Outcome { let root = if input.is_empty() { cfg.workspace.clone() } else { - match workspace::resolve(&cfg.workspace, input) { + match resolve_ws(cfg, input) { Ok(p) => p, Err(e) => return Outcome::err(e), } @@ -1555,7 +1584,7 @@ fn plan_edit( edits: &[Value], cfg: &Config, ) -> Result<(std::path::PathBuf, String, String), String> { - let path = workspace::resolve(&cfg.workspace, input)?; + let path = resolve_ws(cfg, input)?; let content = std::fs::read_to_string(&path).map_err(|e| format!("edit: read {input:?} failed: {e}"))?; let mut new_content = content.clone(); @@ -1639,7 +1668,7 @@ pub fn preview_diff_edit(input: &str, edits: &[Value], cfg: &Config) -> Result Result { - let resolved = workspace::resolve(&cfg.workspace, path)?; + let resolved = resolve_ws(cfg, path)?; let original = std::fs::read_to_string(&resolved).unwrap_or_default(); let new = apply_unified_diff(&original, patch)?; Ok(make_unified_diff(&original, &new, path, 3)) @@ -1648,7 +1677,7 @@ pub fn preview_diff_patch(path: &str, patch: &str, cfg: &Config) -> Result Result { - let path = workspace::resolve(&cfg.workspace, input)?; + let path = resolve_ws(cfg, input)?; let old_content = std::fs::read_to_string(&path).unwrap_or_default(); Ok(make_unified_diff(&old_content, content, input, 3)) } @@ -1891,7 +1920,7 @@ fn apply_patch(args: &Value, cfg: &Config) -> Outcome { if path.is_empty() || patch.is_empty() { return Outcome::err("patch requires 'path' and 'patch'"); } - let resolved = match workspace::resolve(&cfg.workspace, path) { + let resolved = match resolve_ws(cfg, path) { Ok(p) => p, Err(e) => return Outcome::err(e), }; @@ -2028,7 +2057,7 @@ pub async fn execute_diagnostics(args: &Value, cfg: &Config) -> Outcome { let target = if path.is_empty() { cfg.workspace.clone() } else { - match workspace::resolve(&cfg.workspace, path) { + match resolve_ws(cfg, path) { Ok(p) => p, Err(e) => return Outcome::err(e), } @@ -2436,6 +2465,92 @@ fn git_log(args: &Value, cfg: &Config) -> Outcome { } } +/// List OTHER active catalyst-code sessions in this workspace (separate +/// processes), each with its goal, in-progress work, and recently touched +/// files. Awareness only — read-only broadcast of "who is here". Use when +/// something seems off to decide whether a neighbor caused it before assuming +/// you introduced the error. Stale/crashed sessions are auto-pruned by mtime. +fn workspace_activity(_args: &Value, cfg: &Config) -> Outcome { + let my_pid = std::process::id(); + let peers = crate::presence::read_peers(&cfg.workspace, my_pid); + if peers.is_empty() { + return Outcome::ok( + "No other active catalyst-code sessions in this workspace. Any error \ + you are seeing is from your own work or the environment.", + ); + } + let now = crate::presence::unix_now(); + let mut out = format!( + "{} other active session(s) in this workspace:\n", + peers.len() + ); + for p in &peers { + out.push_str(&format!( + "\n- pid {} (started {}, last active {})", + p.pid, + age(now, p.started_at), + age(now, p.last_heartbeat) + )); + if let Some(sid) = &p.session_id { + out.push_str(&format!(", session {sid}")); + } + if let Some(m) = &p.model { + out.push_str(&format!(", model {m}")); + } + if !p.goal.is_empty() { + out.push_str(&format!("\n goal: {}", truncate(p.goal.as_str(), 140))); + } + if !p.in_progress.is_empty() { + out.push_str(&format!("\n in progress: {}", p.in_progress.join("; "))); + } + if !p.next.is_empty() { + out.push_str(&format!("\n next: {}", p.next.join("; "))); + } + if !p.recent_files.is_empty() { + out.push_str(&format!( + "\n recently touched: {}", + p.recent_files.join(", ") + )); + } + if !p.last_activity.is_empty() { + out.push_str(&format!( + "\n last: {}", + truncate(p.last_activity.as_str(), 140) + )); + } + } + Outcome::ok(out) +} + +/// Render a unix-seconds delta as a compact human age ("3m", "2h", "just now"). +fn age(now: u64, then: u64) -> String { + let s = now.saturating_sub(then); + if s < 5 { + "just now".to_string() + } else if s < 60 { + format!("{}s ago", s) + } else if s < 3600 { + format!("{}m ago", s / 60) + } else if s < 86400 { + format!("{}h ago", s / 3600) + } else { + format!("{}d ago", s / 86400) + } +} + +/// Truncate `s` to at most `n` chars, appending an ellipsis if cut. A small +/// local copy of main.rs's `truncate_str` (kept private there) so this module +/// stays self-contained. +fn truncate(s: &str, n: usize) -> String { + if s.chars().count() <= n { + s.to_string() + } else { + let mut t: String = s.chars().take(n.saturating_sub(1)).collect(); + t.push('…'); + t + } +} + fn git_add(args: &Value, cfg: &Config) -> Outcome { let Some(paths) = args.get("paths").and_then(|v| v.as_array()) else { return Outcome::err("git_add requires a 'paths' array"); @@ -2937,6 +3052,43 @@ mod tests { assert!(o.ok, "{}", o.output); } + #[test] + fn never_mode_disables_path_confinement() { + // Under Approval::Never ALL file restrictions are disabled: absolute + // paths and `..` traversal are allowed (the model is fully trusted), so + // path confinement is OFF — not just the dangerous-path list. This is + // the counterpart to `workspace_confines_paths` (which asserts the + // Destructive rejection of the same paths). + use std::sync::atomic::{AtomicU64, Ordering}; + static N: AtomicU64 = AtomicU64::new(0); + let n = N.fetch_add(1, Ordering::SeqCst); + let (_root, mut cfg) = tmp_ws(); + cfg.approval = crate::config::Approval::Never; + // A small file in the PARENT of the workspace, reached both by absolute + // path and by `..` traversal from inside the workspace. + let parent = cfg.workspace.parent().unwrap().to_path_buf(); + let name = format!("catalyst_code_never_out_{n}.txt"); + let outside = parent.join(&name); + fs::write(&outside, "leaked").unwrap(); + // Absolute path: allowed under Never (rejected under Destructive). + let o = execute( + "read_file", + &json!({ "path": outside.to_str().unwrap() }), + &cfg, + ); + assert!( + o.ok, + "absolute read must be allowed under Never: {}", + o.output + ); + assert!(o.output.contains("leaked"), "{}", o.output); + // `..` traversal: allowed under Never. + let o = execute("read_file", &json!({ "path": format!("../{name}") }), &cfg); + assert!(o.ok, "`..` read must be allowed under Never: {}", o.output); + assert!(o.output.contains("leaked"), "{}", o.output); + let _ = fs::remove_file(&outside); + } + #[test] fn read_file_size_guard() { let (_root, cfg) = tmp_ws(); @@ -3473,4 +3625,70 @@ mod tests { ); assert!(!execute("memory", &json!({ "action": "append", "name": "x" }), &cfg).ok); } + + #[test] + fn workspace_activity_lists_peers() { + let (_root, cfg) = tmp_ws(); + let my_pid = std::process::id(); + + // No peers → reassuring "you're alone" message. + let o = execute("workspace_activity", &json!({}), &cfg); + assert!(o.ok, "{}", o.output); + assert!(o.output.contains("No other active"), "{}", o.output); + + // Seed a fake peer (a different pid) in this workspace's presence dir. + let peer_pid = my_pid.wrapping_add(1); + let peer = crate::presence::PresenceRecord::from_work_state( + &crate::WorkState { + goal: "fix CI".into(), + recent_files: vec!["core/src/main.rs".into()], + in_progress: vec!["green build".into()], + ..Default::default() + }, + peer_pid, + Some("peer.json".into()), + None, + crate::presence::unix_now(), + ); + crate::presence::write_presence(&cfg.workspace, peer_pid, &peer); + + let o = execute("workspace_activity", &json!({}), &cfg); + assert!(o.ok, "{}", o.output); + assert!(o.output.contains("1 other active session"), "{}", o.output); + assert!(o.output.contains("fix CI"), "goal missing: {}", o.output); + assert!( + o.output.contains("core/src/main.rs"), + "recent file missing: {}", + o.output + ); + assert!( + o.output.contains("green build"), + "in-progress missing: {}", + o.output + ); + assert!( + o.output.contains(&format!("pid {peer_pid}")), + "pid missing: {}", + o.output + ); + + // Self (my_pid) must never appear even if our own presence file exists. + let me = crate::presence::PresenceRecord::from_work_state( + &crate::WorkState::default(), + my_pid, + None, + None, + crate::presence::unix_now(), + ); + crate::presence::write_presence(&cfg.workspace, my_pid, &me); + let o = execute("workspace_activity", &json!({}), &cfg); + assert!( + !o.output.contains(&format!("pid {my_pid}\n")), + "self leaked: {}", + o.output + ); + + crate::presence::clear_presence(&cfg.workspace, peer_pid); + crate::presence::clear_presence(&cfg.workspace, my_pid); + } } diff --git a/core/src/workspace.rs b/core/src/workspace.rs index c3d1b1d..312b2ed 100644 --- a/core/src/workspace.rs +++ b/core/src/workspace.rs @@ -1,9 +1,16 @@ -// Workspace path confinement. Every file tool resolves paths against a root -// and rejects escapes (absolute paths, `..` traversal, symlinks pointing out). -// bash runs with cwd locked to the root. +// Workspace path confinement. Every file tool resolves paths against a +// root and rejects escapes (absolute paths, `..` traversal, symlinks pointing +// out). bash runs with cwd locked to the root. // Also includes a restricted-path list (.env, .git/**, .ssh/**, id_rsa, …) // that the approval gate uses to PROMPT (under Destructive/Always) rather than // hard-block. Under Approval::Never the list is not enforced at all. +// +// The path CONFINEMENT itself (reject absolute / `..` / symlink-escape) is +// ALSO approval-gated: under `Approval::Never` the file tools call +// `resolve_unconfined` (see tools::resolve_ws) which skips every confinement +// check, so the model may read/write ANY path — absolute, parent-traversing, +// or symlinked-out — matching the "trust the model fully" intent of Never. +// The dangerous-path list is a separate, independent guard (also Never-off). use std::path::{Path, PathBuf}; /// Restricted paths the agent should not read or write without explicit @@ -170,6 +177,24 @@ pub fn resolve(root: &Path, input: &str) -> Result { Ok(canon) } +/// Resolve `input` against `root` WITHOUT path confinement — the untrusted- +/// model guards (absolute-path rejection, `..` traversal rejection, symlink- +/// escape rejection) are all SKIPPED. Used under `Approval::Never`, where the +/// model is fully trusted and ALL file restrictions are disabled. +/// +/// Absolute paths are returned as-is; relative paths are joined to `root` +/// (so `src/foo.rs` still resolves to `/src/foo.rs`, and `../escape` +/// becomes `/../escape`, which the OS resolves naturally when the path is +/// opened). No canonicalization is performed — it was only needed to detect +/// symlink escapes, which are no longer rejected here. +pub fn resolve_unconfined(root: &Path, input: &str) -> Result { + let p = Path::new(input); + if p.is_absolute() { + return Ok(p.to_path_buf()); + } + Ok(root.join(p)) +} + /// True if `path` (already resolved) is confined within `root`. #[allow(dead_code)] pub fn is_confined(root: &Path, path: &Path) -> bool { @@ -246,6 +271,23 @@ mod tests { assert!(check_dangerous_path("sub/.git/HEAD").is_some()); } + #[cfg(unix)] + #[test] + fn unconfined_allows_absolute_and_parent_traversal() { + // Under Approval::Never the file tools use resolve_unconfined: absolute + // paths and `..` traversal are NOT rejected (the model is fully trusted). + let r = tmp_root(); + // Absolute path is returned verbatim (NOT rejected). + let p = resolve_unconfined(&r, "/etc/passwd").unwrap(); + assert_eq!(p, PathBuf::from("/etc/passwd")); + // `..` traversal is allowed — joined to root, OS resolves the `..`. + let p = resolve_unconfined(&r, "../escape").unwrap(); + assert!(p.ends_with("../escape")); + // A normal relative path still resolves under the root. + let p = resolve_unconfined(&r, "sub/b.txt").unwrap(); + assert!(p.starts_with(std::fs::canonicalize(&r).unwrap())); + } + #[cfg(unix)] #[test] fn symlinked_dir_escape_rejected() { diff --git a/install.ps1 b/install.ps1 new file mode 100644 index 0000000..8ed0dc2 --- /dev/null +++ b/install.ps1 @@ -0,0 +1,415 @@ +<# +.SYNOPSIS + Catalyst Code installer for Windows — TUI + optional web service. + +.DESCRIPTION + DEFAULT: download the prebuilt standalone catcode.exe (Rust core embedded) + from GitHub Releases and put it on your user PATH — no compiler, no admin. + With -WithWeb, also download catcode-core.exe + the prebuilt web bundle and + install the web frontend as a Windows Service (NSSM) or a logon Scheduled + Task (delegates to packaging/windows/install-web.ps1). + + No download needed — pipe it straight from the web: + irm https://raw.githubusercontent.com/catalystctl/catcode/master/install.ps1 | iex + + With arguments (e.g. -WithWeb), use the scriptblock form: + & ([scriptblock]::Create((irm https://raw.githubusercontent.com/catalystctl/catcode/master/install.ps1))) -WithWeb + + Or from a repo clone: + pwsh -ExecutionPolicy Bypass -File .\install.ps1 + pwsh -ExecutionPolicy Bypass -File .\install.ps1 -WithWeb + +.PARAMETER Version + Pin a release (e.g. "0.2.0" or "v0.2.0"). Default: latest. + +.PARAMETER BaseUrl + Download base URL override (default: GitHub Releases for the resolved tag). + +.PARAMETER InstallDir + Where catcode.exe + catcode-core.exe are installed. Default: + %LOCALAPPDATA%\Programs\catcode (per-user, no admin). + +.PARAMETER WithWeb + Also install the web frontend service (downloads catcode-core.exe + the + prebuilt web bundle; sets up an NSSM service or a Scheduled Task). + +.PARAMETER Port + Web service port. Default 49283. + +.PARAMETER BindHost + Web bind host. Default 0.0.0.0 (use 127.0.0.1 + a reverse proxy for public use). + +.PARAMETER WebDir + Where to extract the web bundle. Default %LOCALAPPDATA%\catalyst-code\web. + +.PARAMETER WebInstallerUrl + URL to packaging/windows/install-web.ps1 (used only with -WithWeb when this + script is NOT run from a repo clone). Default: raw.githubusercontent.com master. + +.PARAMETER Update + Re-download the latest release and reinstall (also restarts the web service + if it was previously installed). + +.PARAMETER Uninstall + Stop + remove catcode, catcode-core, the web service/task, and install state. + +.PARAMETER DryRun + Print the plan, execute nothing. + +.PARAMETER NoColor + Disable colored output. + +.EXAMPLE + .\install.ps1 + .\install.ps1 -WithWeb -Port 8080 -BindHost 127.0.0.1 + .\install.ps1 -Version 0.2.0 + .\install.ps1 -Update + .\install.ps1 -Uninstall +#> +[CmdletBinding()] +param( + [string]$Version = '', + [string]$BaseUrl = '', + [string]$InstallDir = '', + [switch]$WithWeb, + [int]$Port = 49283, + [string]$BindHost = '0.0.0.0', + [string]$WebDir = '', + [string]$WebInstallerUrl = '', + [switch]$Update, + [switch]$Uninstall, + [switch]$DryRun, + [switch]$NoColor, + [switch]$Help +) + +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' # speed up Invoke-WebRequest on large .exe + +# ── constants + env-derived defaults (resolved in the body so a missing ── +# LOCALAPPDATA never crashes param binding; on Windows it is always set for +# user sessions, but SYSTEM/service accounts may lack it). +$Repo = 'catalystctl/catcode' +$Arch = 'x86_64' +$DefaultWebInstaller = "https://raw.githubusercontent.com/$Repo/master/packaging/windows/install-web.ps1" +function Resolve-LocalAppData { + if ($env:LOCALAPPDATA) { return $env:LOCALAPPDATA } + if ($env:USERPROFILE) { return Join-Path $env:USERPROFILE 'AppData\Local' } + return $env:HOME # non-Windows / fallback +} +$DataDir = Join-Path (Resolve-LocalAppData) 'catalyst-code' +$StateFile = Join-Path $DataDir 'installer.state' +if (-not $InstallDir) { $InstallDir = Join-Path (Resolve-LocalAppData) 'Programs\catcode' } +if (-not $WebDir) { $WebDir = Join-Path $DataDir 'web' } + +# resolve the current PowerShell executable (used to run install-web.ps1 in a +# child process so its exits never kill this installer's flow). +$exeName = if ($PSVersionTable.PSEdition -eq 'Core') { 'pwsh' } else { 'powershell' } +if ($env:OS -eq 'Windows_NT') { $exeName += '.exe' } +$PsExe = Join-Path $PSHOME $exeName + +# mirror the -WithWeb switch into a script-scoped flag (so -Update can set it +# from the recorded install state). +$script:WithWeb = [bool]$WithWeb + +# ── helpers ────────────────────────────────────────────────── +function W-Info($t) { if ($NoColor) { Write-Host " $t" } else { Write-Host " $t" -ForegroundColor Cyan } } +function W-Ok($t) { if ($NoColor) { Write-Host " $t" } else { Write-Host " $t" -ForegroundColor Green } } +function W-Warn($t){ if ($NoColor) { Write-Host " $t" } else { Write-Host " $t" -ForegroundColor Yellow } } +function Die($t) { Write-Host "`n error: $t" -ForegroundColor Red; exit 1 } + +function Show-Help { + $usage = @" + Catalyst Code — installer for Windows + + Usage: + pwsh -ExecutionPolicy Bypass -File .\install.ps1 [options] + irm https://raw.githubusercontent.com/catalystctl/catcode/master/install.ps1 | iex + & ([scriptblock]::Create((irm .../install.ps1))) -WithWeb + + Options: + -Version pin a release (e.g. "0.2.0" or "v0.2.0") default: latest + -BaseUrl download from a mirror instead of GitHub Releases + -InstallDir binary install dir (default: %LOCALAPPDATA%\Programs\catcode) + -WithWeb also install the web frontend service + -Port web service port (default: 49283) + -BindHost web bind host (default: 0.0.0.0) + -WebDir web bundle install dir (default: %LOCALAPPDATA%\catalyst-code\web) + -WebInstallerUrl URL to install-web.ps1 (default: raw.githubusercontent.com master) + -Update re-download latest + reinstall (+ restart the web service) + -Uninstall stop + remove binaries, service, and state + -DryRun print the plan, execute nothing + -NoColor disable colored output + -Help show this help +"@ + Write-Host $usage +} + +# ── release resolution + asset download (mirrors install.sh) ─ +function Resolve-Release { + if ($Version) { + $script:Tag = $Version + if (-not $script:Tag.StartsWith('v')) { $script:Tag = "v$($script:Tag)" } + $script:Ver = $script:Tag.Substring(1) + } else { + $api = "https://api.github.com/repos/$Repo/releases/latest" + try { + $rel = Invoke-RestMethod -Uri $api -Headers @{ 'User-Agent' = 'catcode-installer' } -ErrorAction Stop + $script:Tag = $rel.tag_name + $script:Ver = $script:Tag.Substring(1) + } catch { + Die "could not resolve the latest release from $api.`n The repo may be private or rate-limited. Pass -Version (e.g. -Version 0.2.0) or -BaseUrl to a public mirror." + } + } + if ($BaseUrl) { + $script:Base = $BaseUrl.TrimEnd('/') + } else { + $script:Base = "https://github.com/$Repo/releases/download/$($script:Tag)" + } +} + +# download / + .sha256, verify the checksum. Returns the file path. +function Get-Asset { + param([string]$Name) + $url = "$($script:Base)/$Name" + $dest = Join-Path $env:TEMP $Name + W-Info "Downloading $Name ..." + try { + Invoke-WebRequest -Uri $url -OutFile $dest -UseBasicParsing + } catch { + Die "download failed: $url`n $($_.Exception.Message)" + } + try { + Invoke-WebRequest -Uri "$url.sha256" -OutFile "$dest.sha256" -UseBasicParsing + } catch { + Die "checksum download failed: $url.sha256" + } + $expected = (Get-Content "$dest.sha256" -Raw).Trim().Split(' ')[0].ToLower() + $actual = (Get-FileHash $dest -Algorithm SHA256).Hash.ToLower() + if ($expected -ne $actual) { Die "checksum mismatch for $Name (expected $expected, got $actual)" } + W-Ok "Verified $Name" + return $dest +} + +# ── PATH management ───────────────────────────────────────── +function Add-ToPath { + $path = [Environment]::GetEnvironmentVariable('Path', 'User') + if (-not $path) { $path = '' } + $parts = @($path.Split(';') | Where-Object { $_ -ne '' }) + if ($parts -notcontains $InstallDir) { + $newPath = (($parts + $InstallDir) -join ';') + [Environment]::SetEnvironmentVariable('Path', $newPath, 'User') + W-Ok "Added $InstallDir to your user PATH." + } else { + W-Ok "$InstallDir is already on your user PATH." + } + # refresh the current session so `catcode` works immediately + if ($env:Path -notlike "*$InstallDir*") { $env:Path = "$env:Path;$InstallDir" } +} + +# ── TUI install (download standalone catcode.exe) ──────────── +function Install-Tui { + if (-not (Test-Path -LiteralPath $InstallDir)) { + New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null + } + $tuiAsset = "catcode-$($script:Ver)-windows-$Arch.exe" + $src = Get-Asset $tuiAsset + Copy-Item -LiteralPath $src -Destination (Join-Path $InstallDir 'catcode.exe') -Force + W-Ok "Installed catcode.exe -> $InstallDir\catcode.exe" + Add-ToPath +} + +# ── separate core binary for the web service's CATCODE_CORE ── +function Install-CoreForWeb { + $coreAsset = "catcode-core-$($script:Ver)-windows-$Arch.exe" + $src = Get-Asset $coreAsset + Copy-Item -LiteralPath $src -Destination (Join-Path $InstallDir 'catcode-core.exe') -Force + W-Ok "Installed catcode-core.exe -> $InstallDir\catcode-core.exe" +} + +# ── locate (or download) packaging/windows/install-web.ps1 ─── +function Resolve-WebInstaller { + # 1) local — run from a repo clone (install.ps1 sits at the repo root) + if ($PSScriptRoot) { + $local = Join-Path $PSScriptRoot 'packaging\windows\install-web.ps1' + if (Test-Path -LiteralPath $local) { return $local } + } + # 2) download from -WebInstallerUrl (default: raw master) + $url = if ($WebInstallerUrl) { $WebInstallerUrl } else { $DefaultWebInstaller } + $dest = Join-Path $env:TEMP 'catcode-install-web.ps1' + W-Info "Downloading install-web.ps1 ..." + try { + Invoke-WebRequest -Uri $url -OutFile $dest -UseBasicParsing + } catch { + Die "could not download install-web.ps1 from $url.`n If the repo is private, clone it and run install.ps1 from the repo root, or pass -WebInstallerUrl ." + } + return $dest +} + +# Run install-web.ps1 in a CHILD PROCESS so its exits never terminate this +# installer's flow. Returns the child exit code. +function Invoke-WebInstaller([switch]$DoUninstall) { + $webInstaller = Resolve-WebInstaller + if ($DoUninstall) { + W-Info 'Removing web service (delegating to install-web.ps1) ...' + & $PsExe -NoProfile -File $webInstaller -Uninstall + return $LASTEXITCODE + } + $coreExe = Join-Path $InstallDir 'catcode-core.exe' + W-Info 'Installing web service (delegating to install-web.ps1) ...' + & $PsExe -NoProfile -File $webInstaller -Port $Port -BindHost $BindHost ` + -Version $script:Ver -BaseUrl $script:Base -WebDir $WebDir -CatcodeCore $coreExe + return $LASTEXITCODE +} + +# ── install state ──────────────────────────────────────────── +function Save-State([bool]$WebInstalled) { + $st = [ordered]@{ + version = $script:Ver + with_web = if ($WebInstalled) { 'yes' } else { 'no' } + install_dir = $InstallDir + web_dir = $WebDir + port = $Port + host = $BindHost + installed_at = (Get-Date -Format 'yyyy-MM-ddTHH:mm:ssZ') + } + if (-not (Test-Path -LiteralPath $DataDir)) { New-Item -ItemType Directory -Path $DataDir -Force | Out-Null } + $st | ConvertTo-Json | Set-Content -LiteralPath $StateFile -Encoding UTF8 + W-Ok "Recorded install state -> $StateFile" +} + +function Load-State { + if (-not (Test-Path -LiteralPath $StateFile)) { return $null } + try { return (Get-Content -LiteralPath $StateFile -Raw | ConvertFrom-Json) } catch { return $null } +} + +# ── summaries ──────────────────────────────────────────────── +function Summary-Install { + $webLine = if ($script:WithWeb) { "http://${BindHost}:$Port (service: NSSM or Scheduled Task)" } else { '(not installed — re-run with -WithWeb)' } + Write-Host '' + Write-Host ' ────────────────────────────────────────────' -ForegroundColor Green + Write-Host ' ✓ Installed Catalyst Code v' -NoNewline -ForegroundColor Green + Write-Host "$($script:Ver)" -ForegroundColor Green + Write-Host " binary: $InstallDir\catcode.exe" -ForegroundColor Green + Write-Host " web: $webLine" -ForegroundColor Green + Write-Host ' ────────────────────────────────────────────' -ForegroundColor Green + Write-Host '' + Write-Host ' Open a NEW PowerShell window (so PATH reloads) and run:' -ForegroundColor Green + Write-Host ' catcode' -ForegroundColor Yellow + if ($script:WithWeb) { + Write-Host " web: http://localhost:$Port (logs: $env:LOCALAPPDATA\catalyst-code\catalyst-code-web.log)" -ForegroundColor Green + } + Write-Host ' auth: /login (or set UMANS_API_KEY)' +} + +function Summary-Update { + Write-Host '' + Write-Host ' ────────────────────────────────────────────' -ForegroundColor Green + Write-Host ' ✓ Updated Catalyst Code v' -NoNewline -ForegroundColor Green + Write-Host "$($script:Ver)" -ForegroundColor Green + Write-Host ' ────────────────────────────────────────────' -ForegroundColor Green +} + +function Summary-Uninstall { + Write-Host '' + Write-Host ' ────────────────────────────────────────────' -ForegroundColor Green + Write-Host ' ✓ Removed Catalyst Code' -ForegroundColor Green + Write-Host ' ────────────────────────────────────────────' -ForegroundColor Green + Write-Host ' Open a NEW PowerShell window for a clean PATH.' -ForegroundColor DarkGray +} + +# ── actions ─────────────────────────────────────────────────── +function Do-Install { + Write-Host '' + Write-Host ' Catalyst Code — installer (Windows)' -ForegroundColor Cyan + Write-Host ' mode: download (prebuilt, no compile)' -ForegroundColor DarkGray + Resolve-Release + Write-Host " version: $($script:Ver) base: $($script:Base)" -ForegroundColor DarkGray + Write-Host " install: $InstallDir" -ForegroundColor DarkGray + if ($script:WithWeb) { Write-Host " web: $WebDir (port $Port, host $BindHost)" -ForegroundColor DarkGray } + + if ($DryRun) { + W-Info '[dry-run] would download + install catcode.exe' + if ($script:WithWeb) { W-Info '[dry-run] would also install catcode-core.exe + the web service' } + return + } + + Install-Tui + if ($script:WithWeb) { + # record the TUI install first so a web failure still leaves a usable state + Save-State $false + Install-CoreForWeb + $rc = Invoke-WebInstaller + if ($rc -ne 0) { Die "web service install failed (install-web.ps1 exited $rc)." } + Save-State $true + } else { + W-Info 'Skipping web service (pass -WithWeb to install it)' + Save-State $false + } + Summary-Install +} + +function Do-Update { + Write-Host '' + Write-Host ' Catalyst Code — update' -ForegroundColor Cyan + $st = Load-State + if (-not $st) { Die "no previous install found at $StateFile — run install.ps1 first." } + W-Info "Previous install: v$($st.version) (web: $($st.with_web))" + + Resolve-Release + Write-Host " version: $($script:Ver) base: $($script:Base)" -ForegroundColor DarkGray + + if ($DryRun) { + W-Info '[dry-run] would reinstall catcode.exe' + if ($st.with_web -eq 'yes') { W-Info '[dry-run] would reinstall + restart the web service' } + return + } + + Install-Tui + if ($st.with_web -eq 'yes') { + $script:WithWeb = $true + Install-CoreForWeb + $rc = Invoke-WebInstaller + if ($rc -ne 0) { W-Warn "web service update returned $rc (it self-restarts on re-install)" } + Save-State $true + } else { + Save-State $false + } + Summary-Update +} + +function Do-Uninstall { + Write-Host '' + Write-Host ' Catalyst Code — uninstall' -ForegroundColor Cyan + $st = Load-State + if ($st) { W-Info "Found previous install (v$($st.version), web: $($st.with_web))" } + else { W-Warn "no state file at $StateFile — attempting default paths" } + + if ($DryRun) { + W-Info '[dry-run] would remove the web service + catcode.exe + catcode-core.exe + state' + return + } + + # web service first (if it was installed) + $hadWeb = ($st -and $st.with_web -eq 'yes') + if ($hadWeb) { + $rc = Invoke-WebInstaller -DoUninstall + if ($rc -ne 0) { W-Warn "web uninstall returned $rc (continuing)" } + } + + # binaries + foreach ($b in 'catcode.exe', 'catcode-core.exe') { + $p = Join-Path $InstallDir $b + if (Test-Path -LiteralPath $p) { Remove-Item -LiteralPath $p -Force; W-Ok "Removed $p" } + } + # state + if (Test-Path -LiteralPath $StateFile) { Remove-Item -LiteralPath $StateFile -Force; W-Ok "Removed $StateFile" } + Summary-Uninstall +} + +# ── main ───────────────────────────────────────────────────── +if ($Help) { Show-Help; return } +if ($Update -and $Uninstall) { Die 'cannot combine -Update and -Uninstall.' } +if ($Update) { Do-Update; return } +if ($Uninstall) { Do-Uninstall; return } +Do-Install diff --git a/tui/ask.go b/tui/ask.go index d1aeddc..c1ec8f8 100644 --- a/tui/ask.go +++ b/tui/ask.go @@ -6,8 +6,8 @@ import ( "strings" "github.com/charmbracelet/bubbles/textinput" - "github.com/charmbracelet/lipgloss" tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" ) // askPrompt is the TUI state for a pending `ask` tool call. The core emits an @@ -18,6 +18,11 @@ type askPrompt struct { requestID string questions []askQuestion focusIdx int + // errMsg is a transient inline error (e.g. "Required: …") shown in the + // flyout when submit fails validation. Cleared on the next non-submit + // keypress so it never accumulates in the transcript (the old behavior + // logged a fresh "✗ required" line per Enter, spamming the log). + errMsg string } // askQuestion is one field in the flyout. @@ -180,6 +185,11 @@ func (s *session) handleAskKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { if a == nil { return s, nil } + // Any key other than submit clears a stale inline error so it doesn't + // linger after the user starts fixing the field. + if !s.kb(msg, "send") { + a.errMsg = "" + } // Esc / close: skip the whole prompt (send null). if s.kb(msg, "close") { s.sendAskReply(a, nil) @@ -189,23 +199,28 @@ func (s *session) handleAskKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { if s.kb(msg, "send") { obj, missing := a.answers() if len(missing) > 0 { - s.logError(fmt.Sprintf("required: %s", strings.Join(missing, "; "))) + // Show the error INLINE in the flyout (transient) instead of + // logging to the transcript — repeated Enter on an empty + // required field used to spam "✗ required" lines. + a.errMsg = fmt.Sprintf("Required: %s", strings.Join(missing, "; ")) return s, nil } s.sendAskReply(a, obj) s.logSuccess("↦ answers sent") return s, nil } - // Tab / down: next question (clamp at last). - if s.kb(msg, "next_field") || s.kb(msg, "down") { + // Tab / ↓ / j: next question (clamp at last). The bare "down" fallback + // mirrors the scroll handler so arrows always navigate even if a user + // disabled/rebound nav_down in /keybinds. + if s.kb(msg, "field_next") || msg.String() == "down" || s.kbAny(msg, "nav_down", "nav_down_alt") { if a.focusIdx < len(a.questions)-1 { a.focusIdx++ a.focusInput() } return s, nil } - // Shift+Tab / up: previous question (clamp at first). - if s.kb(msg, "prev_field") || s.kb(msg, "up") { + // Shift+Tab / ↑ / k: previous question (clamp at first). + if s.kb(msg, "field_prev") || msg.String() == "up" || s.kbAny(msg, "nav_up", "nav_up_alt") { if a.focusIdx > 0 { a.focusIdx-- a.focusInput() @@ -338,6 +353,9 @@ func (s *session) renderAskBox() string { } } b.WriteString(footer) + if a.errMsg != "" { + b.WriteString("\n" + errStyle.Render("✗ "+a.errMsg)) + } body := b.String() return lipgloss.NewStyle(). diff --git a/tui/ask_test.go b/tui/ask_test.go new file mode 100644 index 0000000..dc96a2c --- /dev/null +++ b/tui/ask_test.go @@ -0,0 +1,240 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + + tea "github.com/charmbracelet/bubbletea" +) + +// askRequestEvent builds a *coreEvent simulating the core's `ask_request` +// wire event (type + raw JSON carrying request_id + the questions array). +func askRequestEvent(t *testing.T, requestID, questions string) *coreEvent { + t.Helper() + raw, err := json.Marshal(map[string]any{ + "request_id": requestID, + "questions": json.RawMessage(questions), + }) + if err != nil { + t.Fatalf("marshal ask_request: %v", err) + } + return &coreEvent{Type: "ask_request", Raw: raw} +} + +const twoQs = `[ + {"id":"isolation","prompt":"Which isolation?","type":"select","options":["Ephemeral","Persistent"],"required":true}, + {"id":"note","prompt":"Any notes?","type":"text","required":false,"placeholder":"optional"} +]` + +// TestAskRequestSetsFlyout is the core regression: the TUI defined +// parseAskRequest / handleAskKey / renderAskOverlay in ask.go but NEVER wired +// the `ask_request` event into the dispatch switch — so the model's `ask` call +// appeared as a plain tool block ("▸ ask (...)") with NO flyout and the core +// blocked forever on an answer that never came. Now ask_request must populate +// s.pendingAsk with the parsed questions. +func TestAskRequestSetsFlyout(t *testing.T) { + s := initialSession() + s.ready = true + s.width, s.height = 80, 24 + s.layout() + + s.handleCoreEvent(askRequestEvent(t, "ask-1", twoQs)) + + if s.pendingAsk == nil { + t.Fatal("ask_request event must set s.pendingAsk (the flyout was never opened — the original bug)") + } + if s.pendingAsk.requestID != "ask-1" { + t.Fatalf("requestID = %q, want ask-1", s.pendingAsk.requestID) + } + if len(s.pendingAsk.questions) != 2 { + t.Fatalf("expected 2 questions parsed, got %d", len(s.pendingAsk.questions)) + } + q0 := s.pendingAsk.questions[0] + if q0.id != "isolation" || q0.qtype != "select" || len(q0.options) != 2 || !q0.required { + t.Fatalf("first question mis-parsed: %+v", q0) + } + q1 := s.pendingAsk.questions[1] + if q1.id != "note" || q1.qtype != "text" || q1.required { + t.Fatalf("second question mis-parsed: %+v", q1) + } +} + +// TestAskRequestRendersFlyout guards the render wiring (the third missing +// piece): with a pending ask the centered overlay must surface the question +// prompt and select options. Without the renderAskOverlay call in the view +// assembly the flyout state was set but invisible. +func TestAskRequestRendersFlyout(t *testing.T) { + s := initialSession() + s.ready = true + s.width, s.height = 80, 24 + s.layout() + + base := "BASE\nVIEW" + // No pending ask → overlay is a passthrough. + if got := s.renderAskOverlay(base); got != base { + t.Fatalf("renderAskOverlay should be a no-op when nothing is pending; got %q", got) + } + + s.handleCoreEvent(askRequestEvent(t, "ask-1", twoQs)) + if s.pendingAsk == nil { + t.Fatal("setup: ask_request did not set pendingAsk") + } + got := stripANSI(s.renderAskOverlay(base)) + if !strings.Contains(got, "Which isolation?") { + t.Fatalf("ask flyout should render the question prompt; got:\n%s", got) + } + if !strings.Contains(got, "Ephemeral") { + t.Fatalf("ask flyout should render the select options; got:\n%s", got) + } +} + +// TestAskSubmitSendsReplyAndClears guards the key-dispatch wiring: with the +// flyout open, Enter must submit the default selection (first option), dispatch +// ask_reply, and clear the prompt. sendCore is a no-op without a real core. +func TestAskSubmitSendsReplyAndClears(t *testing.T) { + s := initialSession() + s.ready = true + s.width, s.height = 80, 24 + s.layout() + + s.handleCoreEvent(askRequestEvent(t, "ask-1", + `[{"id":"isolation","prompt":"Which?","type":"select","options":["Ephemeral","Persistent"],"required":true}]`)) + if s.pendingAsk == nil { + t.Fatal("setup: ask_request did not set pendingAsk") + } + + // Enter on a select defaults to the first option ("Ephemeral"). + s.handleKey(tea.KeyMsg{Type: tea.KeyEnter}) + if s.pendingAsk != nil { + t.Fatal("Enter should submit the answer and clear s.pendingAsk") + } +} + +// TestAskSkipClears verifies Esc skips the prompt (sends null answers) and +// clears the flyout so the model isn't wedged waiting on ask_reply. +func TestAskSkipClears(t *testing.T) { + s := initialSession() + s.ready = true + s.width, s.height = 80, 24 + s.layout() + + s.handleCoreEvent(askRequestEvent(t, "ask-1", + `[{"id":"isolation","prompt":"Which?","type":"select","options":["A","B"],"required":true}]`)) + if s.pendingAsk == nil { + t.Fatal("setup: ask_request did not set pendingAsk") + } + + s.handleKey(tea.KeyMsg{Type: tea.KeyEsc}) + if s.pendingAsk != nil { + t.Fatal("Esc should skip the ask prompt and clear s.pendingAsk") + } +} + +// TestAskNavigationKeys guards the fix for "up/down don't move between +// questions": handleAskKey used unregistered action names ("next_field"/ +// "down"/"prev_field"/"up") so s.kb returned false for all of them and +// navigation never fired. Now ↓/↑/Tab/Shift+Tab must move focus between +// questions. +func TestAskNavigationKeys(t *testing.T) { + s := initialSession() + s.ready = true + s.width, s.height = 80, 24 + // Use default keybinds so the test is isolated from the user's settings + // (e.g. a user may have disabled nav_up_alt, which would break the k case). + s.keybinds = defaultKeybinds() + s.layout() + + s.handleCoreEvent(askRequestEvent(t, "ask-1", twoQs)) // 2 questions + if s.pendingAsk == nil { + t.Fatal("setup: ask_request did not set pendingAsk") + } + if s.pendingAsk.focusIdx != 0 { + t.Fatalf("focus should start at 0; got %d", s.pendingAsk.focusIdx) + } + + // ↓ moves to question 2. + s.handleKey(tea.KeyMsg{Type: tea.KeyDown}) + if s.pendingAsk.focusIdx != 1 { + t.Fatalf("Down should move focus 0→1; got %d", s.pendingAsk.focusIdx) + } + // ↓ at the last question clamps (no wrap). + s.handleKey(tea.KeyMsg{Type: tea.KeyDown}) + if s.pendingAsk.focusIdx != 1 { + t.Fatalf("Down at last should clamp; got %d", s.pendingAsk.focusIdx) + } + // ↑ moves back to question 1. + s.handleKey(tea.KeyMsg{Type: tea.KeyUp}) + if s.pendingAsk.focusIdx != 0 { + t.Fatalf("Up should move focus 1→0; got %d", s.pendingAsk.focusIdx) + } + // ↑ at the first question clamps. + s.handleKey(tea.KeyMsg{Type: tea.KeyUp}) + if s.pendingAsk.focusIdx != 0 { + t.Fatalf("Up at first should clamp; got %d", s.pendingAsk.focusIdx) + } + // Tab also moves forward. + s.handleKey(tea.KeyMsg{Type: tea.KeyTab}) + if s.pendingAsk.focusIdx != 1 { + t.Fatalf("Tab should move focus 0→1; got %d", s.pendingAsk.focusIdx) + } + // j (nav_down_alt) also moves forward from 0. + s.pendingAsk.focusIdx = 0 + s.handleKey(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("j")}) + if s.pendingAsk.focusIdx != 1 { + t.Fatalf("j should move focus 0→1; got %d", s.pendingAsk.focusIdx) + } + // k (nav_up_alt) moves back. + s.handleKey(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("k")}) + if s.pendingAsk.focusIdx != 0 { + t.Fatalf("k should move focus 1→0; got %d", s.pendingAsk.focusIdx) + } +} + +// TestAskRequiredErrorIsInlineNotSpam guards the UX fix: pressing Enter on an +// empty required text field must NOT log a transcript error line each time +// (the old behavior spammed "✗ required" per Enter). Instead the error is shown +// transiently inside the flyout (a.errMsg) and the prompt stays open. +func TestAskRequiredErrorIsInlineNotSpam(t *testing.T) { + s := initialSession() + s.ready = true + s.width, s.height = 80, 24 + s.layout() + + beforeBlocks := len(s.blocks) + s.handleCoreEvent(askRequestEvent(t, "ask-1", + `[{"id":"feat","prompt":"Name one feature you want the ask tool to support.","type":"text","required":true}]`)) + if s.pendingAsk == nil { + t.Fatal("setup: ask_request did not set pendingAsk") + } + + // Mash Enter 6× on the empty required text field. + for i := 0; i < 6; i++ { + s.handleKey(tea.KeyMsg{Type: tea.KeyEnter}) + } + + // The flyout must still be open (submit was blocked by the empty required field). + if s.pendingAsk == nil { + t.Fatal("Enter on an empty required field must NOT submit / clear the flyout") + } + // The inline error must be set and rendered in the flyout. + if s.pendingAsk.errMsg == "" { + t.Fatal("empty required submit should set an inline a.errMsg") + } + rendered := stripANSI(s.renderAskBox()) + if !strings.Contains(rendered, s.pendingAsk.errMsg) { + t.Fatalf("flyout should render the inline error %q; got:\n%s", s.pendingAsk.errMsg, rendered) + } + // CRITICAL: no transcript error blocks were appended (the old logError spam). + // The ask_request logInfo adds one info block; nothing else should appear. + newBlocks := len(s.blocks) - beforeBlocks + if newBlocks > 1 { + t.Fatalf("repeated Enter must not spam the transcript: %d new blocks (want <=1)", newBlocks) + } + + // Typing clears the stale inline error. + s.handleKey(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("x")}) + if s.pendingAsk.errMsg != "" { + t.Fatal("typing should clear the stale inline error") + } +} diff --git a/tui/blocks.go b/tui/blocks.go index a5eec7f..a437389 100644 --- a/tui/blocks.go +++ b/tui/blocks.go @@ -222,9 +222,24 @@ func (s *session) allTodosComplete() bool { return pend == 0 && run == 0 && done == len(s.todos) } +// maxStoredOutput bounds the tool-result text retained in a block. A multi-MB +// result (e.g. a huge file dump) is stored verbatim though only ~3 lines ever +// render; this caps retention so one result can't pin megabytes of memory for +// the session. The renderer already truncates the visible portion. +const maxStoredOutput = 256 * 1024 // 256 KiB + +// capOutput truncates a stored tool-result string to maxStoredOutput bytes, +// appending a marker when it cut content. +func capOutput(s string) string { + if len(s) <= maxStoredOutput { + return s + } + return s[:maxStoredOutput] + "\n…[truncated]" +} + func (s *session) logToolResult(output string) { b := s.push(blkToolResult) - b.output = output + b.output = capOutput(output) s.refresh() } @@ -689,11 +704,11 @@ func (s *session) rebuildBlocksFromHistory(msgs []map[string]json.RawMessage) { out := contentText(msg["content"]) id := get(msg, "tool_call_id") if b, ok := pending[id]; ok && id != "" { - b.output = out + b.output = capOutput(out) delete(pending, id) } else { b := s.push(blkToolResult) - b.output = out + b.output = capOutput(out) } } } diff --git a/tui/extras_test.go b/tui/extras_test.go new file mode 100644 index 0000000..8d53c32 --- /dev/null +++ b/tui/extras_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "reflect" + "testing" + + "github.com/charmbracelet/bubbles/textinput" + tea "github.com/charmbracelet/bubbletea" +) + +// TestMultilineInputReflectionTargetExists guards the unsafe reflection in +// enableMultilineInput, which swaps textinput's unexported `rsan` sanitizer +// for one that preserves newlines (so Shift+Enter / pasted multi-line text +// survive). The sanitizer field has no public setter, so we write it via +// unsafe.Pointer. If a bubbles upgrade renames or removes `rsan`, the +// function silently degrades to single-line input (no crash) — this test +// makes that regression loud at test time instead of a discovered UX bug. +// +// bubbles is pinned at v1.0.0; if this test fails after an upgrade, either +// re-pin or update enableMultilineInput to the new field name. +func TestMultilineInputReflectionTargetExists(t *testing.T) { + m := textinput.New() + v := reflect.ValueOf(&m).Elem() + f := v.FieldByName("rsan") + if !f.IsValid() { + t.Fatal("textinput.Model no longer has an 'rsan' field — " + + "enableMultilineInput is now a silent no-op; update the reflection or re-pin bubbles") + } + if !f.CanAddr() { + t.Fatal("textinput.Model.rsan is not addressable — " + + "enableMultilineInput's unsafe write cannot proceed") + } +} + +// TestModifiedEnterCSIClassification ensures the reflection in isModifiedEnterCSI +// (which reaches into bubbletea's unexported unknownCSISequenceMsg []byte type) +// classifies correctly and never panics for the ordinary message kinds the TUI +// receives. A bubbles/bubbletea upgrade that changes how modified-Enter arrives +// must not crash the loop or misclassify regular keys. +func TestModifiedEnterCSIClassification(t *testing.T) { + // Non-CSI / ordinary messages: must return false and never panic. + nonCSI := []tea.Msg{ + tea.KeyMsg{Type: tea.KeyEnter}, + tea.KeyMsg{Type: tea.KeyCtrlC}, + tea.MouseMsg{}, + nil, + "not a csi", + []byte("hello"), + []byte(""), + []byte("\x1b[5n"), // a different CSI (device status), not modified-Enter + } + for _, c := range nonCSI { + if isCtrlEnterUnknownCSI(c) { + t.Errorf("ctrl-enter: expected false for %T, got true", c) + } + if isShiftEnterUnknownCSI(c) { + t.Errorf("shift-enter: expected false for %T, got true", c) + } + } + // Modified-Enter CSI byte sequences: must classify true (the function matches + // any []byte whose content is the CSI — it can't see the unexported type). + if !isShiftEnterUnknownCSI([]byte("\x1b[13;2u")) { + t.Error("shift-enter Kitty CSI not recognized") + } + if !isCtrlEnterUnknownCSI([]byte("\x1b[27;5;13~")) { + t.Error("ctrl-enter xterm CSI not recognized") + } +} diff --git a/tui/handlers.go b/tui/handlers.go index 80da791..50a4196 100644 --- a/tui/handlers.go +++ b/tui/handlers.go @@ -35,6 +35,7 @@ func (s *session) accumulateSaved(ev *coreEvent) { func (s *session) handleCoreEvent(ev *coreEvent) tea.Cmd { switch ev.Type { case "ready": + s.coreReady = true // disarm the startup watchdog var models []modelInfo var m map[string]json.RawMessage if err := json.Unmarshal(ev.Raw, &m); err == nil { @@ -58,6 +59,11 @@ func (s *session) handleCoreEvent(ev *coreEvent) tea.Cmd { s.coreBashTimeout = n } } + if raw, ok := m["auto_compact"]; ok { + var b bool + _ = json.Unmarshal(raw, &b) + s.coreAutoCompact = b + } // Provider fields (openai/anthropic endpoints). if raw, ok := m["provider"]; ok { _ = json.Unmarshal(raw, &s.activeProvider) @@ -200,7 +206,7 @@ func (s *session) handleCoreEvent(ev *coreEvent) tea.Cmd { } } if match != nil { - match.output = out + match.output = capOutput(out) match.diff = ev.get("diff") match.ok = ev.get("ok") == "true" match.hasOk = true @@ -270,6 +276,7 @@ func (s *session) handleCoreEvent(ev *coreEvent) tea.Cmd { } case "reset": + s.busy = false // a reset is a conversation boundary — no turn is in flight s.blocks = nil s.cur = nil s.contextTokens = 0 @@ -286,6 +293,13 @@ func (s *session) handleCoreEvent(ev *coreEvent) tea.Cmd { s.logInfo("conversation reset") case "history": + // Loading a session is a conversation boundary — clear any in-flight + // turn/queue so a mid-turn /load or /sessions doesn't wedge the TUI with + // busy=true and a wiped transcript. + s.busy = false + s.cur = nil + s.queuedNext = false + s.queued = nil var m map[string]json.RawMessage if json.Unmarshal(ev.Raw, &m) == nil { if raw, ok := m["messages"]; ok { @@ -303,6 +317,16 @@ func (s *session) handleCoreEvent(ev *coreEvent) tea.Cmd { } } } + case "compacting": + // Pre-compaction warning: the core is about to summarize/drop history. + // Shown as a toast so the pause isn't a mystery (esp. on slow providers + // where the summarize call can take several seconds). + trigger := ev.get("trigger") + if trigger == "" { + trigger = "auto" + } + s.logInfo(fmt.Sprintf("compacting context (%s)…", trigger)) + case "compacted": if ev.get("scope") == "subagent" { break // subagent-internal compaction; don't clutter the main transcript @@ -388,6 +412,24 @@ func (s *session) handleCoreEvent(ev *coreEvent) tea.Cmd { } s.logApproveDiff(ev.get("tool"), ev.get("args"), ev.get("diff")) s.input.Focus() + case "ask_request": + // The model called the `ask` tool and is blocking on the user's + // answers. Parse the questions into a flyout and render it; the core + // waits for `ask_reply` (sent by sendAskReply on submit/skip). rawKey + // is required: ev.get unmarshals into a string, which fails for an + // array and returns "" — the flyout would never open (the original + // bug: ask.go existed but no event case ever called parseAskRequest). + qraw, ok := ev.rawKey("questions") + if !ok { + qraw = json.RawMessage("[]") + } + if a := parseAskRequest(ev.get("request_id"), qraw); a != nil { + s.pendingAsk = a + s.input.Blur() + s.logInfo(fmt.Sprintf("❓ agent asks: %d question%s — answer the prompt", + len(a.questions), pluralS(len(a.questions)))) + s.layout() + } case "intercom_message": // A subagent is prompting the orchestrator for a decision (or a progress // update). need_decision blocks until we reply; progress_update is a log line. @@ -473,7 +515,7 @@ func (s *session) handleCoreEvent(ev *coreEvent) tea.Cmd { // local terminal, which writes its clipboard, so the user can just paste. // Best-effort: terminals that lack OSC 52 (e.g. macOS Terminal.app) ignore // it and the user copies from the hard-wrapped URL shown below instead. - copyToClipboardOSC52(url) + clipCmd := writeOSC52Cmd(url) var b strings.Builder b.WriteString(message) if url != "" { @@ -497,6 +539,7 @@ func (s *session) handleCoreEvent(ev *coreEvent) tea.Cmd { if url != "" { openURL(url) } + return clipCmd // OSC 52 write is a tea.Cmd so it's serialized with the renderer case "steer": // Core acknowledged a steer: the running turn was interrupted and the @@ -550,6 +593,19 @@ func (s *session) handleCoreEvent(ev *coreEvent) tea.Cmd { } } + case "context_breakdown": + // /context reply: parse the token-usage breakdown and open a modal so + // the user can see where the context budget is being spent. + var cb contextBreakdown + if err := json.Unmarshal(ev.Raw, &cb); err != nil { + s.logError("failed to parse context breakdown") + break + } + s.ctxBreakdown = &cb + s.modal.kind = modalContext + s.modal.editing = false + s.modal.fieldIdx = 0 + case "memory_saved": if msg := ev.get("message"); msg != "" { s.logSuccess(msg) @@ -660,7 +716,9 @@ func (s *session) handleCoreEvent(ev *coreEvent) tea.Cmd { func (s *session) applyModels(models []modelInfo) { s.models = models s.modelIdx = 0 - if sel := s.settings.SelectedModel; sel != "" { + if len(models) == 0 { + s.modelIdx = -1 // no model: -1 is an explicit "invalid" sentinel (downstream guards accept it) + } else if sel := s.settings.SelectedModel; sel != "" { for i, mm := range models { if mm.ID == sel || strings.Contains(mm.ID, sel) { s.modelIdx = i @@ -961,9 +1019,13 @@ func (s *session) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { // global: the quit key (default Ctrl+C) quits unless a modal is open // (where esc / ctrl+c closes the modal instead). if s.kb(msg, "quit") && s.modal.kind == modalNone { + // Mark teardown so the core's stdout EOF doesn't trigger an auto-restart, + // then kill the core. The stdout-reader goroutine reaps it via cmd.Wait() + // on EOF — do NOT Wait() here, that would race the reader's Wait on the + // same Cmd (double-reap). + quitting.Store(true) if s.coreCmd != nil && s.coreCmd.Process != nil { _ = s.coreCmd.Process.Kill() - _, _ = s.coreCmd.Process.Wait() // reap the core child so it isn't a zombie } return s, tea.Quit } @@ -971,6 +1033,13 @@ func (s *session) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { if s.modal.kind != modalNone { return s.handleModalKey(msg) } + // ask flyout: a blocking `ask` question is a modal-style overlay that owns + // all keys (option cycling, text entry, submit, skip). Dispatched before + // scrolling/global keys just like the modal above — without it the flyout + // never receives keystrokes even after ask_request set s.pendingAsk. + if s.pendingAsk != nil { + return s.handleAskKey(msg) + } // transcript scrolling works in every state (idle/busy/approval) so the // user can read history while a turn runs or a decision is pending. if s.handleScrollKey(msg) { @@ -1433,9 +1502,17 @@ func (s *session) handleUserLine(text string) tea.Cmd { s.logInfo("dropped last turn") return nil case "/compact": - s.sendCore(map[string]any{"type": "compact"}) + rest := strings.TrimSpace(strings.Join(parts[1:], " ")) + if rest == "" { + s.sendCore(map[string]any{"type": "compact"}) + } else { + s.sendCore(map[string]any{"type": "compact", "instructions": rest}) + } s.logInfo("forcing context compaction…") return nil + case "/context": + s.sendCore(map[string]any{"type": "context"}) + return nil case "/remember": rest := strings.TrimSpace(strings.Join(parts[1:], " ")) if rest == "" { @@ -1648,22 +1725,30 @@ func openURL(url string) { cmd.Stdin = nil cmd.Stdout = nil cmd.Stderr = nil - _ = cmd.Start() + // Run (Start+Wait) in a goroutine so the opener is reaped instead of + // leaving a zombie; fire-and-forget Start() never collects the child. + go func() { _ = cmd.Run() }() } -// copyToClipboardOSC52 writes the OSC 52 escape sequence to set the LOCAL -// terminal's clipboard to text. Over SSH the sequence passes through to the -// user's local terminal, which writes its clipboard — so the user can paste -// (Ctrl/Cmd+V) into their local browser without copying from the (wrapped, -// hard-to-select) transcript. Best-effort: terminals that don't support OSC 52 -// ignore it. The sequence is invisible (no cursor move / no text), so it is -// safe to emit from a Bubble Tea handler between render frames. -func copyToClipboardOSC52(text string) { +// writeOSC52Cmd returns a tea.Cmd that writes the OSC 52 escape sequence to set +// the LOCAL terminal's clipboard to text. Over SSH the sequence passes through +// to the user's local terminal, which writes its clipboard — so the user can +// paste (Ctrl/Cmd+V) into their local browser without copying from the +// (wrapped, hard-to-select) transcript. Best-effort: terminals that don't +// support OSC 52 ignore it. The sequence is invisible (no cursor move / no +// text). Routing the stdout write through a returned tea.Cmd serializes it with +// Bubble Tea's renderer goroutine (which also writes stdout) — a direct +// os.Stdout write from Update races the renderer and can garble the screen. +func writeOSC52Cmd(text string) tea.Cmd { if text == "" { - return + return nil } // OSC 52: ESC ] 52 ; ; BEL. 'c' = the CLIPBOARD // selection (the Ctrl/Cmd+V paste buffer). enc := base64.StdEncoding.EncodeToString([]byte(text)) - os.Stdout.WriteString("\x1b]52;c;" + enc + "\x07") + seq := "\x1b]52;c;" + enc + "\x07" + return func() tea.Msg { + os.Stdout.WriteString(seq) + return nil + } } diff --git a/tui/keybinds.go b/tui/keybinds.go index 50d5790..01deac3 100644 --- a/tui/keybinds.go +++ b/tui/keybinds.go @@ -4,8 +4,8 @@ import ( "fmt" "strings" - "github.com/charmbracelet/lipgloss" tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" ) // --------------------------------------------------------------------------- @@ -126,9 +126,9 @@ func effectiveKeybinds(user map[string]string) map[string]string { // NOT case-folded so an uppercase letter typed into a search filter still // goes to the filter instead of navigating. var caseInsensitiveActions = map[string]bool{ - "approve": true, + "approve": true, "approve_always": true, - "deny": true, + "deny": true, } // kb reports whether msg matches the key bound to action. Single-character keys diff --git a/tui/keybinds_test.go b/tui/keybinds_test.go index 7944b6f..0d3ae78 100644 --- a/tui/keybinds_test.go +++ b/tui/keybinds_test.go @@ -38,10 +38,10 @@ func TestDefaultKeybindsCoverAll(t *testing.T) { // unknown actions are dropped. func TestEffectiveKeybindsMerge(t *testing.T) { user := map[string]string{ - "quit": "ctrl+q", // override - "toggle_reasoning": "ctrl+e", // override + "quit": "ctrl+q", // override + "toggle_reasoning": "ctrl+e", // override "nonexistent_action": "ctrl+z", // unknown — must be dropped - "send": "", // empty — means DISABLED (not default) + "send": "", // empty — means DISABLED (not default) } m := effectiveKeybinds(user) if m["quit"] != "ctrl+q" { diff --git a/tui/login_key_entry_test.go b/tui/login_key_entry_test.go index 0bcfe1e..953351b 100644 --- a/tui/login_key_entry_test.go +++ b/tui/login_key_entry_test.go @@ -19,12 +19,12 @@ func TestLoginKeyEntryEnterCommits(t *testing.T) { // A first-party preset with no key and no OAuth → prompts for inline key. s.providerPresets = []providerPreset{{ - ID: "umans", - Label: "Umans", - Kind: "openai", - BaseURL: "https://example.com/v1", - EnvVar: "UMANS_API_KEY", - HasKey: false, + ID: "umans", + Label: "Umans", + Kind: "openai", + BaseURL: "https://example.com/v1", + EnvVar: "UMANS_API_KEY", + HasKey: false, LoggedIn: false, }} diff --git a/tui/main.go b/tui/main.go index c3adf1e..bb73c8f 100644 --- a/tui/main.go +++ b/tui/main.go @@ -11,6 +11,7 @@ import ( "path/filepath" "runtime" "strings" + "sync/atomic" "syscall" "time" @@ -60,7 +61,11 @@ type session struct { sessionList []sessionEntry skillsList []skillInfo // discoverable skills (drives /skill: autocomplete) coreBashTimeout int + coreAutoCompact bool + ctxBreakdown *contextBreakdown coreRestarts int + coreReady bool // true once the core emitted `ready` (disarms the startup watchdog) + coreStartGen uint64 // bumped each startCore; lets a stale watchdog tick ignore a restart visionModels map[string]bool // user-curated vision-capable model ids (drives /vision) visionModel string // preferred handoff target ("" = pick dynamically) pendingVisionPicker bool // open the vision picker once the config arrives @@ -190,7 +195,7 @@ func coreBinaryPath() string { } sfx := coreExeSuffix() coreName := "catcode-core" + sfx // installed beside the TUI - devName := "core" + sfx // cargo's bin name in the dev build + devName := "core" + sfx // cargo's bin name in the dev build candidates := []string{ "core/target/release/" + devName, "../core/target/release/" + devName, @@ -215,13 +220,24 @@ func coreBinaryPath() string { } // coreProcess holds the running core's *os.Process so a signal handler can -// kill+wait it on SIGHUP/SIGTERM — otherwise closing the terminal (SIGHUP) or -// `kill` (SIGTERM) kills the TUI but orphans catcode-core, which keeps running. -// Set in startCore after cmd.Start(); best-effort only (read under a nil check -// from the signal goroutine, so the benign pointer race is harmless). -var coreProcess *os.Process +// kill it on SIGHUP/SIGTERM — otherwise closing the terminal (SIGHUP) or `kill` +// (SIGTERM) kills the TUI but orphans catcode-core, which keeps running. +// Set in startCore after cmd.Start() (UI thread); read from the signal-handler +// goroutine. An atomic.Pointer is used because the field is shared across +// goroutines — a plain var would be a data race. +var coreProcess atomic.Pointer[os.Process] + +// quitting is set by the signal handler / quit key before killing the core, so +// the coreEOFMsg auto-restart path doesn't spawn a fresh core while the TUI is +// tearing down (a killed core's stdout EOF would otherwise look like a crash). +var quitting atomic.Bool func (s *session) startCore() tea.Cmd { + // Reset startup-tracking state and arm a fresh watchdog generation so a stale + // watchdog tick from a previous (crashed) core is ignored once `ready` lands. + s.coreReady = false + s.coreStartGen++ + gen := s.coreStartGen bin := coreBinaryPath() approval := s.settings.Approval if approval == "" { @@ -270,7 +286,7 @@ func (s *session) startCore() tea.Cmd { return func() tea.Msg { return coreStartErrorMsg{fmt.Errorf("failed to start core (%s): %s", bin, err)} } } s.coreCmd = cmd - coreProcess = cmd.Process // expose to the signal handler (M8): kill+wait on SIGHUP/SIGTERM + coreProcess.Store(cmd.Process) // expose to the signal handler (M8): kill on SIGHUP/SIGTERM s.coreIn = in s.coreEvents = make(chan *coreEvent, 256) s.stdinCh = make(chan []byte, 256) @@ -351,13 +367,35 @@ func (s *session) startCore() tea.Cmd { }() s.sendCore(map[string]any{"type": "init"}) - return waitForEvent(s.coreEvents) + // Arm a startup watchdog: if the core starts but never emits `ready` within + // coreStartupTimeout (e.g. a bad UMANS_CORE path or a config that panics), + // surface a clear error instead of spinning "starting core…" forever. The + // tick carries the generation captured above so a tick from a previous + // (crashed+restarted) core is ignored once `ready` disarms it. + return tea.Batch( + waitForEvent(s.coreEvents), + tea.Tick(coreStartupTimeout, func(time.Time) tea.Msg { return readyTimeoutMsg{gen: gen} }), + ) } // coreStartErrorMsg reports a core subprocess start failure (P1-14: logged on // the UI thread, not from the startCore goroutine). type coreStartErrorMsg struct{ err error } +// coreStartupTimeout is how long startCore's watchdog waits for a `ready` event +// before declaring the core failed to start. +const coreStartupTimeout = 30 * time.Second + +// readyTimeoutMsg is delivered by the startup watchdog when the core has not +// emitted `ready` within coreStartupTimeout. gen ties it to a specific start so +// a tick from a previous (restarted) core is ignored. +type readyTimeoutMsg struct{ gen uint64 } + +// sigtermMsg is sent by the SIGHUP/SIGTERM handler so Bubble Tea restores the +// terminal (alt-screen / raw-mode) via its normal tea.Quit path instead of a +// raw os.Exit that would leave the terminal broken. +type sigtermMsg struct{} + func waitForEvent(ch <-chan *coreEvent) tea.Cmd { return func() tea.Msg { ev, ok := <-ch @@ -431,7 +469,27 @@ func (s *session) Update(msg tea.Msg) (tea.Model, tea.Cmd) { s.logError(msg.err.Error()) return s, nil + case readyTimeoutMsg: + // The startup watchdog fired. Ignore if `ready` already arrived or this + // tick belongs to a previous (restarted) core; otherwise the core never + // came up — surface a clear error instead of spinning forever. + if s.coreReady || msg.gen != s.coreStartGen { + return s, nil + } + s.logError("core did not start within 30s — check UMANS_CORE path / config (Ctrl+C to quit)") + return s, nil + + case sigtermMsg: + // SIGHUP/SIGTERM: restore the terminal via the normal quit path (the + // signal goroutine already killed the core; the reader reaps it). + return s, tea.Quit + case coreEOFMsg: + // A signal-driven teardown (SIGHUP/SIGTERM) or the quit key killed the + // core; the reader then reports EOF. Don't auto-restart — we're quitting. + if quitting.Load() { + return s, tea.Quit + } // Core crashed or exited unexpectedly. Auto-restart once so the user // isn't stranded, and re-auth with the persisted key if we had one. // P1-17: coreRestarts is reset to 0 after every successful turn (see the @@ -527,25 +585,30 @@ func (s *session) Update(msg tea.Msg) (tea.Model, tea.Cmd) { // --------------------------------------------------------------------------- func main() { - // Kill+wait the core child on SIGHUP (terminal closed) / SIGTERM (kill) so it + opts := []tea.ProgramOption{tea.WithAltScreen()} + if loadSettings().MouseWheel { + opts = append(opts, tea.WithMouseCellMotion()) + } + prog := tea.NewProgram(initialSession(), opts...) + + // Kill the core child on SIGHUP (terminal closed) / SIGTERM (kill) so it // isn't orphaned and left running after the TUI exits. Best-effort: a missing - // handle (core not yet started) just exits. + // handle (core not yet started) just sends the quit msg. Instead of os.Exit + // we send a sigtermMsg so Bubble Tea restores the terminal (alt-screen / + // raw-mode) via its normal quit path — os.Exit would leave the terminal + // broken. `quitting` is set first so the core's stdout EOF (from the kill) + // doesn't trigger an auto-restart; the reader goroutine reaps the process. sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGHUP) go func() { <-sigCh - if p := coreProcess; p != nil { + quitting.Store(true) + if p := coreProcess.Load(); p != nil { _ = p.Kill() - _, _ = p.Wait() } - os.Exit(0) + prog.Send(sigtermMsg{}) }() - opts := []tea.ProgramOption{tea.WithAltScreen()} - if loadSettings().MouseWheel { - opts = append(opts, tea.WithMouseCellMotion()) - } - prog := tea.NewProgram(initialSession(), opts...) if _, err := prog.Run(); err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(1) diff --git a/tui/mention.go b/tui/mention.go index fa6ce4f..1771c66 100644 --- a/tui/mention.go +++ b/tui/mention.go @@ -239,28 +239,50 @@ func dirCompletion(query string) []mentionItem { // prefix is microseconds. A short TTL keeps it fresh as files are added. var mentionCache = struct { sync.Mutex - cwd string - list []mentionItem - at time.Time + cwd string + list []mentionItem + at time.Time + walking bool // a background walk for `cwd` is in flight (prevents duplicate/clobbering walks) }{} const mentionCacheTTL = 2 * time.Second const mentionCacheCap = 10000 +// recursiveSearch returns files under the CWD whose path contains the prefix. +// The expensive walk (up to 40k stat calls) runs in a BACKGROUND goroutine so +// it never freezes the UI thread — evalMention reads the cached list and +// returns an empty result while the first walk is in flight. The `walking` flag +// guarantees only one walk per cwd at a time (no duplicate or clobbering fills). func recursiveSearch(prefix string) []mentionItem { cwd, err := os.Getwd() if err != nil { return nil } mentionCache.Lock() + needWalk := false if mentionCache.cwd != cwd || time.Since(mentionCache.at) > mentionCacheTTL { - mentionCache.list = walkMentionList(cwd) - mentionCache.cwd = cwd - mentionCache.at = time.Now() + // Stale or missing: kick a background walk unless one is already running + // for this cwd (a concurrent evalMention may have started it). + if !mentionCache.walking || mentionCache.cwd != cwd { + mentionCache.walking = true + mentionCache.cwd = cwd // claim this cwd so a concurrent call doesn't re-walk + needWalk = true + } } list := mentionCache.list mentionCache.Unlock() + if needWalk { + go fillMentionCache(cwd) + } + + // While the first walk is in flight the cache is empty — return nothing so + // the flyout stays open (the next keystroke re-evals against the filled + // cache). On a large repo the walk completes well within a few keystrokes. + if len(list) == 0 { + return nil + } + lp := strings.ToLower(prefix) var items []mentionItem for _, it := range list { @@ -275,6 +297,20 @@ func recursiveSearch(prefix string) []mentionItem { return items } +// fillMentionCache walks the CWD once and stores the result. Runs in a goroutine +// so it never blocks the UI thread; only commits if the cwd hasn't changed +// under us (a cd race), and always clears the in-progress flag. +func fillMentionCache(cwd string) { + walked := walkMentionList(cwd) + mentionCache.Lock() + defer mentionCache.Unlock() + if mentionCache.cwd == cwd { + mentionCache.list = walked + mentionCache.at = time.Now() + } + mentionCache.walking = false +} + // walkMentionList walks the CWD once, collecting non-ignored entries (capped at // mentionCacheCap) with relative paths. Hidden files and heavy dirs are pruned. func walkMentionList(cwd string) []mentionItem { diff --git a/tui/mention_test.go b/tui/mention_test.go index ce081c1..c433bbc 100644 --- a/tui/mention_test.go +++ b/tui/mention_test.go @@ -3,6 +3,7 @@ package main import ( "strings" "testing" + "time" tea "github.com/charmbracelet/bubbletea" ) @@ -42,15 +43,26 @@ func TestMentionRecursiveFilter(t *testing.T) { if !s.mentionActive { t.Fatal("flyout should be active after @main") } + // recursiveSearch fills its cache from a background goroutine (a + // synchronous walk would freeze the UI on large repos). Poll until the + // walk completes and main.go appears in the flyout. + deadline := time.Now().Add(2 * time.Second) idx := -1 - for i, it := range s.mentionItems { - if it.display == "main.go" { - idx = i + for { + for i, it := range s.mentionItems { + if it.display == "main.go" { + idx = i + break + } + } + if idx >= 0 { break } - } - if idx < 0 { - t.Fatalf("flyout should contain main.go; got %v", itemsDisplay(s.mentionItems)) + if time.Now().After(deadline) { + t.Fatalf("flyout should contain main.go after walk completes; got %v", itemsDisplay(s.mentionItems)) + } + s.evalMention() // re-eval against the now-populated cache + time.Sleep(5 * time.Millisecond) } // Move the cursor to the main.go entry and accept with Tab. s.mentionCursor = idx diff --git a/tui/modal.go b/tui/modal.go index 19e2a09..b364bcd 100644 --- a/tui/modal.go +++ b/tui/modal.go @@ -35,6 +35,7 @@ const ( modalLogout modalKeybinds modalOauthCode + modalContext ) type modal struct { @@ -406,10 +407,11 @@ func (s *session) commandItems() []listItem { {label: "/reset", desc: "wipe conversation + session file"}, {label: "/clear", desc: "clear view (keep session file)"}, {label: "/undo", desc: "drop last turn"}, - {label: "/compact", desc: "force context compaction"}, + {label: "/compact", desc: "force compaction (opt: /compact )"}, {label: "/sessions", desc: "open session picker"}, {label: "/new", desc: "start a fresh session file"}, {label: "/stats", desc: "token + turn totals"}, + {label: "/context", desc: "token-usage breakdown (top consumers)"}, {label: "/abort", desc: "stop running turn (or Esc)"}, {label: "/steer", desc: "steer an in-flight turn (or Ctrl+Enter)"}, {label: "/settings", desc: "open settings modal"}, @@ -596,6 +598,12 @@ func (s *session) handleModalKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { return s.handleSettingsKey(msg) case modalHelp: return s.handleHelpKey(msg) + case modalContext: + // Display-only modal: enter or esc dismisses it. + if msg.String() == "enter" || s.kb(msg, "select") || s.kbAny(msg, "close", "quit") { + s.closeModal() + } + return s, nil } return s, nil } @@ -948,6 +956,7 @@ func (s *session) settingsFields() []settingsField { {label: "Reasoning", value: s.settings.ReasoningEffort, hint: "enter to cycle"}, {label: "Theme", value: activeTheme.name, hint: "enter to pick"}, {label: "Bash Timeout", value: fmt.Sprintf("%ds", s.coreBashTimeout), hint: "enter to edit"}, + {label: "Auto Compact", value: boolStr(s.coreAutoCompact), hint: "enter to toggle"}, {label: "Sandbox", value: s.settings.Sandbox, hint: "enter to cycle"}, {label: "No Network", value: boolStr(s.settings.NoNetwork), hint: "enter to toggle"}, {label: "Mouse Wheel", value: boolStr(s.settings.MouseWheel), hint: "enter to toggle"}, @@ -963,6 +972,18 @@ func boolStr(b bool) string { return "off" } +// humanTokens renders a token count compactly (e.g. 1.2k, 47k, 128k) for the +// /context breakdown modal. +func humanTokens(n uint64) string { + if n < 1000 { + return fmt.Sprintf("%d", n) + } + if n < 1_000_000 { + return fmt.Sprintf("%.1fk", float64(n)/1000) + } + return fmt.Sprintf("%.1fM", float64(n)/1_000_000) +} + // settingsFieldIndex returns the index of the settings field whose label // matches, or -1 if none. Palette shortcuts (/key, /approval) use this so // they target the correct row regardless of the field ordering in @@ -1033,6 +1054,10 @@ func (s *session) activateField(idx int) (tea.Model, tea.Cmd) { s.settings.NoNetwork = !s.settings.NoNetwork _ = s.settings.save() s.logInfo(fmt.Sprintf("no-network: %s (applies on next launch)", boolStr(s.settings.NoNetwork))) + case "Auto Compact": + s.coreAutoCompact = !s.coreAutoCompact + s.sendCore(map[string]any{"type": "set_config", "key": "auto_compact", "value": s.coreAutoCompact}) + s.logInfo(fmt.Sprintf("auto-compact: %s", boolStr(s.coreAutoCompact))) case "Mouse Wheel": s.settings.MouseWheel = !s.settings.MouseWheel _ = s.settings.save() @@ -1467,6 +1492,8 @@ func (s *session) renderModalBody() string { return s.renderHelpModal() case modalKeybinds: return s.renderKeybindsModal() + case modalContext: + return s.renderContextModal() } return "" } @@ -1613,6 +1640,67 @@ func (s *session) renderListModal(title string, items []listItem, showFilter boo return modalBox(w, body) } +// renderContextModal renders the /context token-usage breakdown: total/window, +// per-role buckets, and the top token consumers. Read-only display. +func (s *session) renderContextModal() string { + w := s.modalWidth(78) + var lines []string + lines = append(lines, accentStyle.Render("◆ Context Breakdown")) + lines = append(lines, separatorStyle.Render(strings.Repeat("─", w-2))) + cb := s.ctxBreakdown + if cb == nil { + lines = append(lines, mutedStyle.Render(" no data")) + lines = append(lines, "") + lines = append(lines, dimStyle.Render(" esc close")) + return modalBox(w, strings.Join(lines, "\n")) + } + lines = append(lines, fmt.Sprintf("%s: %s / %s (%s%%)", + baseStyle.Render("Total"), + accentStyle.Render(humanTokens(cb.Total)), + mutedStyle.Render(humanTokens(cb.Window)), + accentStyle.Render(fmt.Sprintf("%d", cb.Pct)))) + lines = append(lines, fmt.Sprintf("%s: %d", baseStyle.Render("Messages"), cb.Messages)) + // Per-role buckets. + if len(cb.ByRole) > 0 { + lines = append(lines, "") + lines = append(lines, dimStyle.Render(" by role:")) + // Stable order: system, user, assistant, tool, then any others. + order := []string{"system", "user", "assistant", "tool"} + seen := map[string]bool{} + for _, r := range order { + if v, ok := cb.ByRole[r]; ok { + lines = append(lines, fmt.Sprintf(" %-9s %s", r, humanTokens(v))) + seen[r] = true + } + } + for r, v := range cb.ByRole { + if !seen[r] { + lines = append(lines, fmt.Sprintf(" %-9s %s", r, humanTokens(v))) + } + } + } + // Top consumers. + if len(cb.TopConsumers) > 0 { + lines = append(lines, "") + lines = append(lines, dimStyle.Render(" top consumers:")) + for _, c := range cb.TopConsumers { + prev := c.Preview + maxRunes := w - 34 + if maxRunes < 20 { + maxRunes = 20 + } + if len([]rune(prev)) > maxRunes { + prev = string([]rune(prev)[:maxRunes]) + "…" + } + lines = append(lines, fmt.Sprintf(" #%d %-9s %s %s", + c.Index, c.Role, humanTokens(c.Tokens), mutedStyle.Render(prev))) + } + } + lines = append(lines, "") + lines = append(lines, dimStyle.Render(" esc close")) + return modalBox(w, strings.Join(lines, "\n")) +} + func (s *session) renderSettingsModal() string { w := s.modalWidth(72) fields := s.settingsFields() diff --git a/tui/pin_test.go b/tui/pin_test.go index ea002e4..df5553e 100644 --- a/tui/pin_test.go +++ b/tui/pin_test.go @@ -13,7 +13,7 @@ import ( type nopWriteCloser struct{} func (nopWriteCloser) Write(p []byte) (int, error) { return len(p), nil } -func (nopWriteCloser) Close() error { return nil } +func (nopWriteCloser) Close() error { return nil } // wireCoreStub gives the session a capture channel for sendCore commands. func wireCoreStub(s *session) { diff --git a/tui/protocol.go b/tui/protocol.go index 680f4d8..088aae8 100644 --- a/tui/protocol.go +++ b/tui/protocol.go @@ -95,6 +95,26 @@ type memoryEntry struct { Tags []string `json:"tags"` } +// contextConsumer is one row of the core's "context_breakdown" event +// top_consumers array (the biggest token consumers in the conversation). +type contextConsumer struct { + Index int `json:"index"` + Role string `json:"role"` + Tokens uint64 `json:"tokens"` + Preview string `json:"preview"` +} + +// contextBreakdown mirrors the core's "context_breakdown" event payload so the +// TUI can render a /context modal showing where the context budget is spent. +type contextBreakdown struct { + Total uint64 `json:"total_tokens"` + Window uint64 `json:"context_window"` + Pct uint64 `json:"pct"` + Messages int `json:"messages"` + ByRole map[string]uint64 `json:"by_role"` + TopConsumers []contextConsumer `json:"top_consumers"` +} + // skillInfo mirrors one element of the core's "skills" event array. The // content (SKILL.md body) is sent by the core so /skill: can apply a // skill without the read_file path restriction blocking global skills. diff --git a/tui/render.go b/tui/render.go index 1079034..a653ca9 100644 --- a/tui/render.go +++ b/tui/render.go @@ -853,7 +853,10 @@ func (s *session) View() string { parts = append(parts, s.renderInputBox(), s.renderFooter()) view := strings.Join(parts, "\n") if s.modal.kind != modalNone { - return s.renderModalOverlay(view) + view = s.renderModalOverlay(view) } - return view + // ask flyout: a blocking `ask` prompt renders as a centered overlay on top + // of the full view (like the modal above). renderAskOverlay is a no-op + // (returns base unchanged) when s.pendingAsk is nil. + return s.renderAskOverlay(view) } diff --git a/tui/settings.go b/tui/settings.go index 7dd15c3..a40d3fe 100644 --- a/tui/settings.go +++ b/tui/settings.go @@ -214,9 +214,37 @@ func (s *settingsStore) save() error { if err != nil { return err } - tmp := s.path + ".tmp" - if err := os.WriteFile(tmp, data, 0600); err != nil { + // Unique temp file (random suffix via os.CreateTemp) in the SAME directory + // as the target, so two processes saving settings concurrently never + // share a temp file — a shared temp would interleave writes and rename a + // corrupted file over settings.json. Atomic rename within one filesystem + // is preserved (same dir). + base := filepath.Base(s.path) + f, err := os.CreateTemp(dir, "."+base+".*.tmp") + if err != nil { + return err + } + tmp := f.Name() + if _, err := f.Write(data); err != nil { + f.Close() + os.Remove(tmp) + return err + } + if err := f.Sync(); err != nil { + f.Close() + os.Remove(tmp) + return err + } + f.Close() + // 0600: settings.json may hold API keys. CreateTemp already uses 0600 on + // Unix, but set it explicitly for parity with the original WriteFile path. + if err := os.Chmod(tmp, 0600); err != nil { + os.Remove(tmp) + return err + } + if err := os.Rename(tmp, s.path); err != nil { + os.Remove(tmp) return err } - return os.Rename(tmp, s.path) + return nil } diff --git a/web/src/components/chat.tsx b/web/src/components/chat.tsx index abd0f91..85bff18 100644 --- a/web/src/components/chat.tsx +++ b/web/src/components/chat.tsx @@ -121,8 +121,14 @@ export function Chat() { if (window.confirm("Reset the conversation and session file? This cannot be undone.")) return a.reset(); return; - case "compact": - return a.compact(); + case "compact": { + const instr = window.prompt( + "Optional: what should compaction preserve?\n(e.g. “Focus on code samples and API usage”)\nLeave blank for the default summary.", + ); + return a.compact(instr?.trim() || undefined); + } + case "context": + return a.context(); case "new": return a.newSession(); case "abort": diff --git a/web/src/lib/commands.ts b/web/src/lib/commands.ts index b110e9b..7163850 100644 --- a/web/src/lib/commands.ts +++ b/web/src/lib/commands.ts @@ -24,8 +24,9 @@ export const COMMANDS: CommandDef[] = [ { label: "/reset", desc: "wipe conversation + session file", category: "session", action: "reset" }, { label: "/clear", desc: "clear view (keep session file)", category: "session", action: "clear" }, { label: "/undo", desc: "drop last turn", category: "session", action: "undo" }, - { label: "/compact", desc: "force context compaction", category: "session", action: "compact" }, + { label: "/compact", desc: "force compaction (opt: instructions)", category: "session", action: "compact" }, { label: "/stats", desc: "token + turn totals", category: "session", action: "stats" }, + { label: "/context", desc: "token-usage breakdown (top consumers)", category: "session", action: "context" }, { label: "/abort", desc: "stop running turn", category: "session", action: "abort", streaming: true }, // ── Config ── diff --git a/web/src/lib/reducer.ts b/web/src/lib/reducer.ts index 024742c..eb85d63 100644 --- a/web/src/lib/reducer.ts +++ b/web/src/lib/reducer.ts @@ -466,6 +466,15 @@ export function reduce(state: AgentState, ev: AgentEvent): AgentState { `Agent asks: ${ev.questions.length} question${ev.questions.length === 1 ? "" : "s"}`, ), }; + case "compacting": + return { + ...state, + toasts: pushToast( + state.toasts, + "info", + `Compacting context${ev.trigger ? ` (${ev.trigger})` : ""}…`, + ), + }; case "compacted": return { ...state, @@ -499,6 +508,20 @@ export function reduce(state: AgentState, ev: AgentEvent): AgentState { stats: ev, currentSessionFile: ev.session_file || state.currentSessionFile, }; + case "context_breakdown": { + const top = (ev.top_consumers ?? []) + .slice(0, 3) + .map((c) => `${c.role} #${c.index}: ${c.tokens.toLocaleString()}`) + .join(" · "); + return { + ...state, + toasts: pushToast( + state.toasts, + "info", + `Context: ${ev.total_tokens.toLocaleString()} / ${ev.context_window.toLocaleString()} tokens (${ev.pct}%)${top ? ` — top: ${top}` : ""}`, + ), + }; + } case "history": return { ...state, diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 09e7752..e952834 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -269,7 +269,9 @@ export type CoreEvent = | { type: "ask_request"; request_id: string; questions: AskQuestion[] } | { type: "metrics" } & Metrics | { type: "umans_conc"; used: number | null; limit: number | null; provider: string } - | { type: "compacted"; before_tokens: number; after_tokens: number } + | { type: "compacted"; before_tokens: number; after_tokens: number; summary_chars?: number } + | { type: "compacting"; before_tokens: number; trigger: string } + | { type: "context_breakdown"; total_tokens: number; context_window: number; pct: number; messages: number; system_tokens: number; by_role: Record; top_consumers: { index: number; role: string; tokens: number; preview: string }[] } | { type: "http_retry"; attempt?: number; status?: number; backoff_ms?: number; reason?: string } | { type: "sessions"; sessions: SessionEntry[]; files: string[] } | Stats @@ -321,7 +323,8 @@ export type CoreCommand = | { type: "abort" } | { type: "reset" } | { type: "clear" } - | { type: "compact" } + | { type: "compact"; instructions?: string } + | { type: "context" } | { type: "approve"; request_id: string; decision: "yes" | "no" | "always" } | { type: "set_approval"; mode: "never" | "destructive" | "always" } | { type: "set_key"; api_key: string; provider?: string } diff --git a/web/src/lib/use-agent.ts b/web/src/lib/use-agent.ts index 285aa2c..ef7dae2 100644 --- a/web/src/lib/use-agent.ts +++ b/web/src/lib/use-agent.ts @@ -57,9 +57,10 @@ export interface AgentApi { newSession: () => Promise; loadSession: (path: string) => Promise; listSessions: () => Promise; - compact: () => Promise; + compact: (instructions?: string) => Promise; reset: () => Promise; stats: () => Promise; + context: () => Promise; dismissToast: (id: string) => void; // ── Subagent / intercom ── intercomReply: (reply: string) => Promise; @@ -394,9 +395,14 @@ export function useAgent(): AgentApi { [switchToSession], ); const listSessions = useCallback(() => send({ type: "list_sessions" }), [send]); - const compact = useCallback(() => send({ type: "compact" }), [send]); + const compact = useCallback( + (instructions?: string) => + send({ type: "compact", ...(instructions ? { instructions } : {}) }), + [send], + ); const reset = useCallback(() => send({ type: "reset" }), [send]); const stats = useCallback(() => send({ type: "stats" }), [send]); + const context = useCallback(() => send({ type: "context" }), [send]); const dismissToast = useCallback((id: string) => { setState((s) => reduce(s, { type: "_dismiss_toast", id })); @@ -630,6 +636,7 @@ export function useAgent(): AgentApi { compact, reset, stats, + context, dismissToast, intercomReply, askReply,