Skip to content

feat(otel): emit server-side spans + traceresponse header (RIG-2889) #713

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mattwilkinsonn merged 2 commits into from
Aug 29, 2026
Merged

feat(otel): emit server-side spans + traceresponse header (RIG-2889) #713

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
02358a7
feat(otel): emit server-side spans + traceresponse header (RIG-2889)
rigel-mintaka Aug 28, 2026
84de91f
fix(otel): flush server providers on drain + real disabled-path asser…
rigel-mintaka Aug 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion flake.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
# touched (guest-image/default.nix:82-87). vendorHash pins the fetched set —
# the whole module graph, so it matches guestd's proxyVendor hash. Recompute
# with lib.fakeHash on a go.mod/go.sum move.
vendorHash = "sha256-di6nYpDDZblu6TPpendW88l5LMRZNzSE2t2tiWfms9c=";
vendorHash = "sha256-GHZsEfvnu1tY6Bd7Fxg7SEWEI+HS0NlQuBbm6pz/UK4=";
in
{
packages = forAllSystems (
Expand Down
37 changes: 37 additions & 0 deletions go/cmd/compass-server/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"syscall"
"time"

"github.com/RigelBuild/compass/go/internal/otel"
"github.com/RigelBuild/compass/go/server"
)

Expand Down Expand Up @@ -91,6 +92,39 @@ func run() error {
stopDrainLog := logOnDrainSignal()
defer stopDrainLog()

// OTel emission (T4b): endpoint-gated off the ENV-only knob. When
// OTEL_EXPORTER_OTLP_ENDPOINT is empty, Setup* install no provider and return
// no-op shutdowns, so this is zero-overhead on the shipped socket-only path.
otelCfg := otel.Config{
ServiceName: "compass-server",
ServiceVersion: version,
Endpoint: cfg.OtelEndpoint,
}
traceShutdown, err := otel.SetupTracerProvider(ctx, otelCfg)
if err != nil {
return fmt.Errorf("otel: tracer provider: %w", err)
}
// The drain ctx is already cancelled by the time these defers fire (the signal
// that ends Serve is the same one that cancels ctx), so a raw ctx.Shutdown
// would abort its final ForceFlush and drop the last batch. Sever the
// cancellation and bound the flush at 2s (design.md: mirror the agent's 2s
// shutdown bound), derived HERE at fire time so the deadline is not consumed
// by the process lifetime.
defer func() {
sctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
defer cancel()
_ = traceShutdown(sctx) // best-effort flush on drain; a collector error here is not actionable at exit
}()
meterShutdown, err := otel.SetupMeterProvider(ctx, otelCfg)
if err != nil {
return fmt.Errorf("otel: meter provider: %w", err)
}
defer func() {
sctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
defer cancel()
_ = meterShutdown(sctx) // best-effort flush on drain; a collector error here is not actionable at exit
}()

return server.Serve(ctx, cfg)
}

Expand Down Expand Up @@ -193,6 +227,9 @@ func buildServeConfig(args []string) (server.ServeConfig, bool, error) {
AdminHandle: *f.adminHandle,
CORSAllowedOrigin: *f.corsAllowedOrigin,
PublicURL: firstNonEmpty(*f.publicURL, os.Getenv("COMPASS_PUBLIC_URL")),
// ENV-ONLY knob (Matt 2026-08-28): the OTLP exporter and the enable-gate
// read one source, so no --otel-endpoint flag. Empty = tracing off.
OtelEndpoint: os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"),
}, false, nil
}

Expand Down
3 changes: 2 additions & 1 deletion go/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ go 1.25.0
require (
connectrpc.com/connect v1.20.0
connectrpc.com/cors v0.1.0
connectrpc.com/otelconnect v0.9.0
github.com/BurntSushi/toml v1.6.0
github.com/cachix/secretspec/secretspec-go v0.15.0
github.com/hashicorp/golang-lru/v2 v2.0.7
Expand All @@ -34,6 +35,7 @@ require (
go.opentelemetry.io/otel v1.46.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0
go.opentelemetry.io/otel/metric v1.46.0
go.opentelemetry.io/otel/sdk v1.46.0
go.opentelemetry.io/otel/sdk/metric v1.46.0
go.opentelemetry.io/otel/trace v1.46.0
Expand Down Expand Up @@ -89,7 +91,6 @@ require (
github.com/zeebo/xxh3 v1.1.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect
go.opentelemetry.io/otel/metric v1.46.0 // indirect
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
golang.org/x/crypto v0.55.0 // indirect
golang.org/x/sys v0.47.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ connectrpc.com/connect v1.20.0 h1:6TNDAB+WeNd2uolWNlYczB5E0KNNaVMNUEx8JEUsPmQ=
connectrpc.com/connect v1.20.0/go.mod h1:A2ygJrukXwWy32vkCAAHNVguZrqZ+jeZ9rGRnGR4dN4=
connectrpc.com/cors v0.1.0 h1:f3gTXJyDZPrDIZCQ567jxfD9PAIpopHiRDnJRt3QuOQ=
connectrpc.com/cors v0.1.0/go.mod h1:v8SJZCPfHtGH1zsm+Ttajpozd4cYIUryl4dFB6QEpfg=
connectrpc.com/otelconnect v0.9.0 h1:NggB3pzRC3pukQWaYbRHJulxuXvmCKCKkQ9hbrHAWoA=
connectrpc.com/otelconnect v0.9.0/go.mod h1:AEkVLjCPXra+ObGFCOClcJkNjS7zPaQSqvO0lCyjfZc=
github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78=
Expand Down
10 changes: 10 additions & 0 deletions go/internal/comms/comms.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import (
"fmt"

"connectrpc.com/connect"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"

"github.com/RigelBuild/compass/go/events"
compassv1 "github.com/RigelBuild/compass/go/gen/compass/v1"
Expand Down Expand Up @@ -349,6 +351,10 @@ func (c *Comms) PostMessage(
if err != nil {
return nil, edgeError(err)
}
// Stamp the appended message's id onto the handler's RPC span (the
// otelconnect origin span mounted on this service), so a trace filters to
// one message. A no-op when no span is active (no provider installed).
trace.SpanFromContext(ctx).SetAttributes(attribute.String("compass.message.id", string(msg.ID)))
// Publish only on a genuine insert: an idempotent retry returns the stored
// row unchanged (inserted=false), so re-fanning MessagePosted would emit a
// spurious live state-change for a row that did not change.
Expand Down Expand Up @@ -386,6 +392,10 @@ func (c *Comms) RespondToAsk(
if err != nil {
return nil, edgeError(err)
}
// Stamp the answer message's id onto the handler's RPC span (the delivery
// origin for the answer post), matching PostMessage. A no-op when no span is
// active.
trace.SpanFromContext(ctx).SetAttributes(attribute.String("compass.message.id", string(answerMsg.ID)))
// MessageUpdated carries the ask's new answered state to the UI; it is NOT a
// delivery trigger. MessagePosted for the answer message IS the delivery
// trigger — it fans out on the normal message rail, so an offline or
Expand Down
8 changes: 7 additions & 1 deletion go/server/cors_pgtest_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ import (
"net/http/httptest"
"testing"

"connectrpc.com/otelconnect"

"github.com/RigelBuild/compass/go/events"
compassv1 "github.com/RigelBuild/compass/go/gen/compass/v1"
"github.com/RigelBuild/compass/go/gen/compass/v1/compassv1connect"
Expand Down Expand Up @@ -67,7 +69,11 @@ func buildDoorHandler(t *testing.T, corsOrigin string) http.Handler {
CORSAllowedOrigin: corsOrigin,
}
secretsSvc := newSecretsService(st, nil, nil)
srv, err := buildNetworkServer(ctx, cfg, svc, commsSvc, secretsSvc, nil, st, admin, nil, nil)
otelIC, err := otelconnect.NewInterceptor()
if err != nil {
t.Fatalf("otelconnect.NewInterceptor: %v", err)
}
srv, err := buildNetworkServer(ctx, cfg, svc, commsSvc, secretsSvc, nil, st, admin, nil, nil, otelIC)
if err != nil {
t.Fatalf("buildNetworkServer: %v", err)
}
Expand Down
14 changes: 13 additions & 1 deletion go/server/network_door.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@ import (

"connectrpc.com/connect"
connectcors "connectrpc.com/cors"
"connectrpc.com/otelconnect"
"github.com/rs/cors"

"github.com/RigelBuild/compass/go/gen/compass/v1/compassv1connect"
"github.com/RigelBuild/compass/go/internal/auth"
"github.com/RigelBuild/compass/go/internal/gen/compass/v1/compassv1internalconnect"
"github.com/RigelBuild/compass/go/internal/otel"
"github.com/RigelBuild/compass/go/internal/runnerhub"
"github.com/RigelBuild/compass/go/internal/secrets"
"github.com/RigelBuild/compass/go/internal/store"
Expand Down Expand Up @@ -140,7 +142,7 @@ func networkCORS(origin string) *cors.Cors {
AllowedOrigins: []string{origin},
AllowedMethods: connectcors.AllowedMethods(),
AllowedHeaders: append(connectcors.AllowedHeaders(), "Authorization"),
ExposedHeaders: connectcors.ExposedHeaders(),
ExposedHeaders: append(connectcors.ExposedHeaders(), "traceresponse"),
AllowCredentials: false,
})
}
Expand Down Expand Up @@ -241,6 +243,7 @@ func buildNetworkServer(
adminID store.AccountID,
netTLS *tls.Config,
resolver secrets.Resolver,
otelIC *otelconnect.Interceptor,
) (*http.Server, error) {
handle := cfg.resolvedAdminHandle()
stateDir := cfg.StateDir
Expand All @@ -260,7 +263,16 @@ func buildNetworkServer(
slog.Info("network door bootstrap admin token written",
"path", tokenPath, "handle", handle, "listen", cfg.Listen)

// otelconnect (outermost) produces the RPC span and NewTraceResponseInterceptor
// stamps the traceresponse header, prepended to the shared bearer + admin-gate
// chain so every network-door service (CompassService, CommsService, and
// SecretsService, which rides the same chain) carries them. Both are inert
// no-ops when no provider is installed (empty OtelEndpoint). Ordering: otel
// first keeps the security-critical Bearer→AdminGate order unchanged relative
// to itself.
interceptors := connect.WithInterceptors(
otelIC,
otel.NewTraceResponseInterceptor(),
auth.BearerInterceptor(st),
auth.BearerStreamInterceptor(st),
auth.NewAdminGate(adminID),
Expand Down
Loading
Loading