Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 57 additions & 0 deletions Taskfile.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -422,6 +422,44 @@ tasks:
cmds:
- go run ./cmd/calipers

beat:rattle:
desc: "Run rattle in isolated single-beat mode against Prometheus: task beat:rattle [WATCH=config/dev/rattle/watch.yaml] [QUERY=config/dev/rattle/query.yaml] [PROM_URL=http://localhost:9090]"
vars:
WATCH: '{{.WATCH | default "config/dev/rattle/watch.yaml"}}'
QUERY: '{{.QUERY | default "config/dev/rattle/query.yaml"}}'
PROM_URL: '{{.PROM_URL | default "http://localhost:9090"}}'
cmds:
- go run ./cmd/calipers step rattle --watch {{.WATCH}} --query-config {{.QUERY}} --prom-url {{.PROM_URL}}

beat:clank:
desc: "Run clank in isolated single-beat mode against a detection: task beat:clank DETECTION=<path> [PROFILE=config/dev] [MODEL=haiku] (key-gated)"
requires:
vars: [DETECTION]
vars:
PROFILE: '{{.PROFILE | default "config/dev"}}'
MODEL: '{{.MODEL | default "haiku"}}'
cmds:
- go run ./cmd/calipers step clank --detection {{.DETECTION}} --profile {{.PROFILE}} --model {{.MODEL}}

beat:hiss:
desc: "Run hiss in isolated single-beat mode against a proposal: task beat:hiss PROPOSAL=<path> [POLICY=config/dev/hiss/policy.yaml]"
requires:
vars: [PROPOSAL]
vars:
POLICY: '{{.POLICY | default "config/dev/hiss/policy.yaml"}}'
cmds:
- go run ./cmd/calipers step hiss --proposal {{.PROPOSAL}} --policy {{.POLICY}}

beat:thump:
desc: "Run thump in isolated single-beat mode against a decision: task beat:thump DECISION=<path> [CATALOG=config/dev/actions/catalog.yaml] [DRY_RUN=true]"
requires:
vars: [DECISION]
vars:
CATALOG: '{{.CATALOG | default "config/dev/actions/catalog.yaml"}}'
DRY_RUN: '{{.DRY_RUN | default "true"}}'
cmds:
- go run ./cmd/calipers step thump --decision {{.DECISION}} --catalog {{.CATALOG}} --dry-run={{.DRY_RUN}}

# docs/dev-environment.md — the fully-local dev environment. dev:cluster is
# the one piece Tilt itself can't do: `allow_k8s_contexts()` and `helm()`
# both evaluate at Tiltfile load, so the k3d context has to exist before
Expand Down Expand Up @@ -564,6 +602,25 @@ tasks:
- kubectl --context k3d-thump-dev -n thump delete job dev-govern-{{.RUN_ID}} configmap dev-govern-input-{{.RUN_ID}} --ignore-not-found
- rm -rf bin/govern/{{.RUN_ID}}

dev:pipeline:
desc: "Run headless end-to-end pipeline locally without cluster: task dev:pipeline DETECTION=<path> [PROFILE=config/dev] [MODEL=haiku] (key-gated)"
requires:
vars: [DETECTION]
vars:
PROFILE: '{{.PROFILE | default "config/dev"}}'
MODEL: '{{.MODEL | default "haiku"}}'
cmds:
- go run ./cmd/calipers pipeline --detection {{.DETECTION}} --profile {{.PROFILE}} --model {{.MODEL}}

dev:mock:
desc: "Run lightweight telemetry and NATS mock server: task dev:mock [PROM_PORT=9090] [NATS=false] [NATS_PORT=4222]"
vars:
PROM_PORT: '{{.PROM_PORT | default "9090"}}'
NATS_PORT: '{{.NATS_PORT | default "4222"}}'
NATS: '{{.NATS | default "false"}}'
cmds:
- go run ./cmd/calipers mock --prom-port {{.PROM_PORT}} {{if eq .NATS "true"}}--nats --nats-port {{.NATS_PORT}}{{end}}

dev:probe:
desc: >-
Fire N real clank reasoning runs against a captured detection, straight
Expand Down
43 changes: 42 additions & 1 deletion cmd/calipers/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"

"github.com/google/go-cmp/cmp"
Expand Down Expand Up @@ -66,7 +67,7 @@ func run(t *testing.T, dir string, args ...string) (stdout, stderr string, exitC
// test drives the compiled binary as a subprocess, not the package) so it
// is pinned here too; a change to calipers.go's topUsage without a matching
// change here fails this test rather than passing silently.
const wantTopUsage = "usage: calipers <incidents|approve|force|unseal|corpus|rca|tune|replay|harvest|probe|transcript|scorecard|validate> [flags]\n"
const wantTopUsage = "usage: calipers <incidents|approve|force|unseal|corpus|rca|tune|replay|harvest|probe|transcript|scorecard|validate|step|pipeline|mock> [flags]\n"

func TestMain_ReturnsUsageAndExitCodeTwoForBadInvocations(t *testing.T) {
t.Parallel()
Expand Down Expand Up @@ -120,3 +121,43 @@ func TestMain_RoutesArgsAndStreamsCorrectlyOnASuccessfulVerb(t *testing.T) {
t.Error("wrong stdout", diff)
}
}

func TestMain_SubprocessStepAndPipelineUsageErrors(t *testing.T) {
t.Parallel()
dir := t.TempDir()

tests := map[string]struct {
args []string
wantCode int
wantErrContains string
}{
"step with no subverb exits 2 with step usage": {
args: []string{"step"},
wantCode: 2,
wantErrContains: "usage: calipers step <rattle|clank|hiss|thump> [flags]",
},
"pipeline with no flags exits 2 with pipeline usage": {
args: []string{"pipeline"},
wantCode: 2,
wantErrContains: "usage: calipers pipeline --detection <path>",
},
}

for name, tc := range tests {
t.Run(name, func(t *testing.T) {
t.Parallel()

stdout, stderr, exitCode := run(t, dir, tc.args...)

if diff := cmp.Diff(tc.wantCode, exitCode); diff != "" {
t.Errorf("wrong exit code (-want +got):\n%s", diff)
}
if !strings.Contains(stderr, tc.wantErrContains) {
t.Errorf("stderr does not contain %q, got: %s", tc.wantErrContains, stderr)
}
if diff := cmp.Diff("", stdout); diff != "" {
t.Errorf("stdout not empty (-want +got):\n%s", diff)
}
})
}
}
69 changes: 69 additions & 0 deletions docs/dev-environment.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,75 @@ a separate repo under `~/projects/ceph/`; `dev` is provisioned by this one, via
you. A five-beat cycle on Haiku costs fractions of a cent; nothing about running this
environment burns meaningful spend.

## Isolated beat testing & lightweight development (zero cluster / low resource)

When iterating on a single beat, testing policy logic, or running reasoning without provisioning a k3d cluster or Docker VM, `calipers` provides single-beat execution steps, a headless end-to-end pipeline runner, and a lightweight mock server.

### Single-beat execution

Drive individual beats directly from fixtures without NATS or Kubernetes:

| Beat | Task / Command | Input | Output |
|---|---|---|---|
| rattle | `task beat:rattle [PROM_URL=http://localhost:9090]` | PromQL queries against Prometheus | `signal.Detection` JSON |
| clank | `task beat:clank DETECTION=<path> [MODEL=haiku]` | Detection fixture + profile configs | `proposal.Set` JSON |
| hiss | `task beat:hiss PROPOSAL=<path> [POLICY=<path>]` | Proposal fixture + policy YAML | `decision.Governed` JSON |
| thump | `task beat:thump DECISION=<path> [CATALOG=<path>] [DRY_RUN=true]` | Decision fixture + catalog YAML | `outcome.Outcome` JSON |

Equivalent direct CLI invocations:

```sh
# rattle: detect SLO burn against Prometheus (or dev:mock)
go run ./cmd/calipers step rattle --watch config/dev/rattle/watch.yaml --query-config config/dev/rattle/query.yaml --prom-url http://localhost:9090

# clank: reason over a captured detection (requires ANTHROPIC_API_KEY)
go run ./cmd/calipers step clank --detection test/fixtures/detections/cart-failure.json --profile config/dev --model haiku

# hiss: evaluate governance policy against a proposal
go run ./cmd/calipers step hiss --proposal test/fixtures/proposals/cart-failure.json --policy config/dev/hiss/policy.yaml

# thump: dry-run action synthesis and safety bounds against a decision
go run ./cmd/calipers step thump --decision test/fixtures/decisions/cart-failure.json --catalog config/dev/actions/catalog.yaml --dry-run=true
```

### Headless pipeline execution

Run the full five-beat loop in-process against local configuration profiles and live reasoning:

```sh
task dev:pipeline DETECTION=test/fixtures/detections/cart-failure.json
```

Or via CLI:

```sh
go run ./cmd/calipers pipeline --detection test/fixtures/detections/cart-failure.json --profile config/dev --model haiku
```

This runs detection ingestion, clank reasoning, hiss policy governance, and thump dry-run actuation in a single process without NATS or Kubernetes, printing the complete `pipeline.Result` JSON to stdout.

### Lightweight mock telemetry & broker

For local testing without the 12 GB substrate, `dev:mock` stands up an in-process HTTP stub for Prometheus/Loki and an optional embedded NATS JetStream server:

```sh
task dev:mock PROM_PORT=9090
```

With embedded NATS:

```sh
task dev:mock PROM_PORT=9090 NATS=true NATS_PORT=4222
```

Direct CLI invocation:

```sh
go run ./cmd/calipers mock --prom-port 9090 --nats --nats-port 4222
```

The mock telemetry server returns deterministic PromQL vector and matrix responses (such as `slo:current_burn_rate:ratio`), fake Loki query streams, and Kubernetes pod list stubs for telemetry tools in clank and rattle.

## Bringing it up

```sh
Expand Down
Loading