feat(otel): emit runner client spans (RIG-2890)

flake.nix

@@ -49,7 +49,7 @@
       # touched (guest-image/default.nix:82-87). vendorHash pins the fetched set —
       # the whole module graph, so it matches guestd's proxyVendor hash. Recompute
       # with lib.fakeHash on a go.mod/go.sum move.
-      vendorHash = "sha256-di6nYpDDZblu6TPpendW88l5LMRZNzSE2t2tiWfms9c=";
+      vendorHash = "sha256-GHZsEfvnu1tY6Bd7Fxg7SEWEI+HS0NlQuBbm6pz/UK4=";
     in
     {
       packages = forAllSystems (

go/cmd/compass-runner/main.go

@@ -18,10 +18,12 @@ import (
 	"os/signal"
 	"strings"
 	"syscall"
+	"time"
 
 	"connectrpc.com/connect"
 
 	"github.com/RigelBuild/compass/go/internal/agentuid"
+	"github.com/RigelBuild/compass/go/internal/otel"
 	"github.com/RigelBuild/compass/go/internal/runner"
 	"github.com/RigelBuild/compass/go/internal/runtime"
 )
@@ -37,7 +39,7 @@ func main() {
 	}
 }
 
-func run() error {
+func run() error { //nolint:funlen // flag registration + operator-input validation is the honest bulk; the env-only OTel setup call tips it 2 lines over — extracting the flags would scatter ~15 flag vars for no readability gain
 	runnerID := flag.String("runner-id", "",
 		"This Runner's stable id, cross-checked against the token subject. Defaults to $COMPASS_RUNNER_ID.")
 	serverAddr := flag.String("server", "",
@@ -158,6 +160,13 @@ func run() error {
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 	defer stop()
 
+	// OTel emission (env-only, bounded flush on drain); see setupOtel.
+	otelShutdown, err := setupOtel(ctx)
+	if err != nil {
+		return err
+	}
+	defer otelShutdown()
+
 	return runner.Run(ctx, runner.RunnerConfig{
 		RunnerID:   id,
 		ServerAddr: addr,
@@ -169,6 +178,41 @@ func run() error {
 	}, specs, log)
 }
 
+// setupOtel installs the tracer and meter providers off the env-only OTLP
+// endpoint, returning one shutdown that flushes both. When
+// OTEL_EXPORTER_OTLP_ENDPOINT is empty the providers are no-ops and the shutdown
+// is a no-op, so tracing is off with zero overhead. The export gate lives here in
+// the global-provider install — the runner's outbound otelconnect interceptor is
+// mounted unconditionally and is inert against the no-op global.
+func setupOtel(ctx context.Context) (shutdown func(), err error) {
+	cfg := otel.Config{
+		ServiceName:    "compass-runner",
+		ServiceVersion: version,
+		Endpoint:       os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"),
+	}
+	tracerShutdown, err := otel.SetupTracerProvider(ctx, cfg)
+	if err != nil {
+		return nil, fmt.Errorf("otel: tracer provider: %w", err)
+	}
+	meterShutdown, err := otel.SetupMeterProvider(ctx, cfg)
+	if err != nil {
+		_ = tracerShutdown(ctx) // roll back the tracer provider we just installed; nothing actionable on its error here
+		return nil, fmt.Errorf("otel: meter provider: %w", err)
+	}
+	// The drain ctx is already cancelled by the time this fires (the signal that
+	// ends runner.Run is the same one that cancels ctx), so a raw ctx.Shutdown
+	// would abort its final ForceFlush and drop the last batch. Sever the
+	// cancellation and bound the flush at 2s (design.md: mirror the agent's 2s
+	// shutdown bound), derived at fire time so the deadline is not consumed by the
+	// process lifetime — matching the context.WithoutCancel precedent in run.go.
+	return func() {
+		sctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
+		defer cancel()
+		_ = tracerShutdown(sctx) // best-effort flush on shutdown; export error not actionable at exit
+		_ = meterShutdown(sctx)  // best-effort flush on shutdown; export error not actionable at exit
+	}, nil
+}
+
 // backendFlags holds the runtime-backend selection flags, registered on the
 // default flag set before flag.Parse and resolved into a runtime after it.
 type backendFlags struct {

go/cmd/compass-server/main.go

@@ -19,6 +19,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/RigelBuild/compass/go/internal/otel"
 	"github.com/RigelBuild/compass/go/server"
 )
 
@@ -91,6 +92,39 @@ func run() error {
 	stopDrainLog := logOnDrainSignal()
 	defer stopDrainLog()
 
+	// OTel emission (T4b): endpoint-gated off the ENV-only knob. When
+	// OTEL_EXPORTER_OTLP_ENDPOINT is empty, Setup* install no provider and return
+	// no-op shutdowns, so this is zero-overhead on the shipped socket-only path.
+	otelCfg := otel.Config{
+		ServiceName:    "compass-server",
+		ServiceVersion: version,
+		Endpoint:       cfg.OtelEndpoint,
+	}
+	traceShutdown, err := otel.SetupTracerProvider(ctx, otelCfg)
+	if err != nil {
+		return fmt.Errorf("otel: tracer provider: %w", err)
+	}
+	// The drain ctx is already cancelled by the time these defers fire (the signal
+	// that ends Serve is the same one that cancels ctx), so a raw ctx.Shutdown
+	// would abort its final ForceFlush and drop the last batch. Sever the
+	// cancellation and bound the flush at 2s (design.md: mirror the agent's 2s
+	// shutdown bound), derived HERE at fire time so the deadline is not consumed
+	// by the process lifetime.
+	defer func() {
+		sctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
+		defer cancel()
+		_ = traceShutdown(sctx) // best-effort flush on drain; a collector error here is not actionable at exit
+	}()
+	meterShutdown, err := otel.SetupMeterProvider(ctx, otelCfg)
+	if err != nil {
+		return fmt.Errorf("otel: meter provider: %w", err)
+	}
+	defer func() {
+		sctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
+		defer cancel()
+		_ = meterShutdown(sctx) // best-effort flush on drain; a collector error here is not actionable at exit
+	}()
+
 	return server.Serve(ctx, cfg)
 }
 
@@ -193,6 +227,9 @@ func buildServeConfig(args []string) (server.ServeConfig, bool, error) {
 		AdminHandle:       *f.adminHandle,
 		CORSAllowedOrigin: *f.corsAllowedOrigin,
 		PublicURL:         firstNonEmpty(*f.publicURL, os.Getenv("COMPASS_PUBLIC_URL")),
+		// ENV-ONLY knob (Matt 2026-08-28): the OTLP exporter and the enable-gate
+		// read one source, so no --otel-endpoint flag. Empty = tracing off.
+		OtelEndpoint: os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"),
 	}, false, nil
 }
 

go/go.mod

@@ -17,6 +17,7 @@ go 1.25.0
 require (
 	connectrpc.com/connect v1.20.0
 	connectrpc.com/cors v0.1.0
+	connectrpc.com/otelconnect v0.9.0
 	github.com/BurntSushi/toml v1.6.0
 	github.com/cachix/secretspec/secretspec-go v0.15.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
@@ -34,6 +35,7 @@ require (
 	go.opentelemetry.io/otel v1.46.0
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0
+	go.opentelemetry.io/otel/metric v1.46.0
 	go.opentelemetry.io/otel/sdk v1.46.0
 	go.opentelemetry.io/otel/sdk/metric v1.46.0
 	go.opentelemetry.io/otel/trace v1.46.0
@@ -89,7 +91,6 @@ require (
 	github.com/zeebo/xxh3 v1.1.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect
-	go.opentelemetry.io/otel/metric v1.46.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.11.0 // indirect
 	golang.org/x/crypto v0.55.0 // indirect
 	golang.org/x/sys v0.47.0 // indirect

go/go.sum

@@ -2,6 +2,8 @@ connectrpc.com/connect v1.20.0 h1:6TNDAB+WeNd2uolWNlYczB5E0KNNaVMNUEx8JEUsPmQ=
 connectrpc.com/connect v1.20.0/go.mod h1:A2ygJrukXwWy32vkCAAHNVguZrqZ+jeZ9rGRnGR4dN4=
 connectrpc.com/cors v0.1.0 h1:f3gTXJyDZPrDIZCQ567jxfD9PAIpopHiRDnJRt3QuOQ=
 connectrpc.com/cors v0.1.0/go.mod h1:v8SJZCPfHtGH1zsm+Ttajpozd4cYIUryl4dFB6QEpfg=
+connectrpc.com/otelconnect v0.9.0 h1:NggB3pzRC3pukQWaYbRHJulxuXvmCKCKkQ9hbrHAWoA=
+connectrpc.com/otelconnect v0.9.0/go.mod h1:AEkVLjCPXra+ObGFCOClcJkNjS7zPaQSqvO0lCyjfZc=
 github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
 github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78=

go/internal/comms/comms.go

@@ -21,6 +21,8 @@ import (
 	"fmt"
 
 	"connectrpc.com/connect"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/RigelBuild/compass/go/events"
 	compassv1 "github.com/RigelBuild/compass/go/gen/compass/v1"
@@ -349,6 +351,10 @@ func (c *Comms) PostMessage(
 	if err != nil {
 		return nil, edgeError(err)
 	}
+	// Stamp the appended message's id onto the handler's RPC span (the
+	// otelconnect origin span mounted on this service), so a trace filters to
+	// one message. A no-op when no span is active (no provider installed).
+	trace.SpanFromContext(ctx).SetAttributes(attribute.String("compass.message.id", string(msg.ID)))
 	// Publish only on a genuine insert: an idempotent retry returns the stored
 	// row unchanged (inserted=false), so re-fanning MessagePosted would emit a
 	// spurious live state-change for a row that did not change.
@@ -386,6 +392,10 @@ func (c *Comms) RespondToAsk(
 	if err != nil {
 		return nil, edgeError(err)
 	}
+	// Stamp the answer message's id onto the handler's RPC span (the delivery
+	// origin for the answer post), matching PostMessage. A no-op when no span is
+	// active.
+	trace.SpanFromContext(ctx).SetAttributes(attribute.String("compass.message.id", string(answerMsg.ID)))
 	// MessageUpdated carries the ask's new answered state to the UI; it is NOT a
 	// delivery trigger. MessagePosted for the answer message IS the delivery
 	// trigger — it fans out on the normal message rail, so an offline or

go/internal/runner/otel_test.go

@@ -0,0 +1,120 @@
+//go:build unix
+
+package runner
+
+// OTel wiring tests for the Runner-side seam: Dial's outbound RunnerService
+// client mounts the otelconnect interceptor, which emits a CLIENT span per RPC
+// when a global tracer provider is installed — and none when disabled, since the
+// empty-endpoint path installs no global provider (the export gate lives in the
+// provider install, not in RunnerConfig).
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/RigelBuild/compass/go/internal/gen/compass/v1/compassv1internalconnect"
+	"go.opentelemetry.io/otel"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	"go.opentelemetry.io/otel/trace"
+	"go.opentelemetry.io/otel/trace/noop"
+)
+
+// enrollServerURL stands up an h2c httptest RunnerService serving enrollStub and
+// returns its base URL, torn down via t.Cleanup — so a test can drive Dial (which
+// builds the interceptor-wrapped client) end to end against a real dial.
+func enrollServerURL(t *testing.T) string {
+	t.Helper()
+	path, handler := compassv1internalconnect.NewRunnerServiceHandler(enrollStub{})
+	mux := http.NewServeMux()
+	mux.Handle(path, handler)
+	srv := httptest.NewUnstartedServer(mux)
+	srv.Config.Protocols = cleartextHTTP2()
+	srv.Start()
+	t.Cleanup(srv.Close)
+	return srv.URL
+}
+
+// installInMemoryTracer installs an SDK tracer provider backed by an in-memory
+// recorder as the global provider (the source otelconnect.NewInterceptor reads),
+// restoring the prior global on cleanup. It returns the recorder.
+func installInMemoryTracer(t *testing.T) *tracetest.SpanRecorder {
+	t.Helper()
+	rec := tracetest.NewSpanRecorder()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(rec))
+	prev := otel.GetTracerProvider()
+	otel.SetTracerProvider(tp)
+	t.Cleanup(func() {
+		otel.SetTracerProvider(prev)
+		_ = tp.Shutdown(context.Background()) // best-effort flush; test root ctx, error not actionable
+	})
+	return rec
+}
+
+// clientSpanCount counts CLIENT-kind spans among the recorded spans.
+func clientSpanCount(spans []sdktrace.ReadOnlySpan) int {
+	n := 0
+	for _, s := range spans {
+		if s.SpanKind() == trace.SpanKindClient {
+			n++
+		}
+	}
+	return n
+}
+
+// TestDialEmitsClientSpanWhenEnabled asserts Dial's outbound client emits a
+// CLIENT span on the Enroll RPC when a global tracer provider is installed.
+func TestDialEmitsClientSpanWhenEnabled(t *testing.T) {
+	rec := installInMemoryTracer(t)
+	url := enrollServerURL(t)
+
+	// context.Background() is the test root context.
+	if _, err := Dial(context.Background(), RunnerConfig{
+		RunnerID:   "r-1",
+		ServerAddr: url,
+		Token:      "tok",
+		HTTPClient: h2cHTTPClient(t),
+		// Emission gated by the installed global tracer provider, not any config
+		// field — installInMemoryTracer set one above.
+	}); err != nil {
+		t.Fatalf("Dial err = %v, want nil", err)
+	}
+
+	if got := clientSpanCount(rec.Ended()); got == 0 {
+		t.Fatalf("enabled: client spans = %d, want >= 1 (otelconnect emits a CLIENT span per RPC)", got)
+	}
+}
+
+// TestDialEmitsNoClientSpanWhenDisabled asserts that with no SDK provider
+// installed (the empty-endpoint disabled path), the same dial records no spans —
+// the otelconnect interceptor is a no-op against the global noop provider.
+func TestDialEmitsNoClientSpanWhenDisabled(t *testing.T) {
+	// Pin the global to a noop provider (the disabled-path state: SetupTracerProvider
+	// installs nothing when the endpoint is empty), and record via a separate SDK
+	// provider that is NOT global — so any span the interceptor emits would be caught,
+	// yet none is, because otelconnect reads the (noop) global.
+	prev := otel.GetTracerProvider()
+	otel.SetTracerProvider(noop.NewTracerProvider())
+	t.Cleanup(func() { otel.SetTracerProvider(prev) })
+
+	rec := tracetest.NewSpanRecorder()
+	sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(rec)) // deliberately not made global
+
+	url := enrollServerURL(t)
+
+	// context.Background() is the test root context.
+	if _, err := Dial(context.Background(), RunnerConfig{
+		RunnerID:   "r-1",
+		ServerAddr: url,
+		Token:      "tok",
+		HTTPClient: h2cHTTPClient(t),
+	}); err != nil {
+		t.Fatalf("Dial err = %v, want nil", err)
+	}
+
+	if got := clientSpanCount(rec.Ended()); got != 0 {
+		t.Fatalf("disabled: client spans = %d, want 0 (no global SDK provider installed)", got)
+	}
+}

go/internal/runner/runner.go

@@ -20,6 +20,7 @@ import (
 	"net/http"
 
 	"connectrpc.com/connect"
+	"connectrpc.com/otelconnect"
 
 	compassv1internal "github.com/RigelBuild/compass/go/internal/gen/compass/v1"
 	"github.com/RigelBuild/compass/go/internal/gen/compass/v1/compassv1internalconnect"
@@ -103,9 +104,15 @@ func Dial(ctx context.Context, cfg RunnerConfig) (*ServerLink, error) {
 	if httpClient == nil {
 		httpClient = http.DefaultClient
 	}
+	otelInterceptor, err := otelconnect.NewInterceptor()
+	if err != nil {
+		return nil, fmt.Errorf("otel: connect interceptor: %w", err)
+	}
 	client := compassv1internalconnect.NewRunnerServiceClient(
 		httpClient, cfg.ServerAddr,
-		connect.WithInterceptors(&bearerToken{token: cfg.Token}),
+		// otelconnect goes first (outermost) so enroll/Sessions dials emit
+		// client spans; it is a no-op when no global provider is installed.
+		connect.WithInterceptors(otelInterceptor, &bearerToken{token: cfg.Token}),
 	)
 	resp, err := client.Enroll(ctx, connect.NewRequest(&compassv1internal.EnrollRequest{
 		RunnerId: cfg.RunnerID,

go/server/cors_pgtest_test.go

@@ -28,6 +28,8 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"connectrpc.com/otelconnect"
+
 	"github.com/RigelBuild/compass/go/events"
 	compassv1 "github.com/RigelBuild/compass/go/gen/compass/v1"
 	"github.com/RigelBuild/compass/go/gen/compass/v1/compassv1connect"
@@ -67,7 +69,11 @@ func buildDoorHandler(t *testing.T, corsOrigin string) http.Handler {
 		CORSAllowedOrigin: corsOrigin,
 	}
 	secretsSvc := newSecretsService(st, nil, nil)
-	srv, err := buildNetworkServer(ctx, cfg, svc, commsSvc, secretsSvc, nil, st, admin, nil, nil)
+	otelIC, err := otelconnect.NewInterceptor()
+	if err != nil {
+		t.Fatalf("otelconnect.NewInterceptor: %v", err)
+	}
+	srv, err := buildNetworkServer(ctx, cfg, svc, commsSvc, secretsSvc, nil, st, admin, nil, nil, otelIC)
 	if err != nil {
 		t.Fatalf("buildNetworkServer: %v", err)
 	}