From 6a02310c73d35e0cb44f79300c273363c7d0366f Mon Sep 17 00:00:00 2001 From: Edgar Babajanyan Date: Mon, 6 Jul 2026 13:34:57 -0700 Subject: [PATCH 1/2] One-command Kubernetes + AWS deployment (deploy/), multi-arch image publishing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deploy/kubernetes — kustomize tree for the serverless topology: - serving: StatefulSet with a PVC per replica (a warm disk makes restarts a seconds-long delta catch-up; two replicas sharing one PVC would corrupt each other, hence StatefulSet not Deployment), startupProbe sized for cold rebuilds (the engine binds the port only AFTER loading collections — without it a big rebuild gets liveness-killed into a crash loop), fsGroupChangePolicy: OnRootMismatch (no recursive chown of warm caches) - writer: stateless Deployment, emptyDir scratch, kill-any-time - components/cold: optional serve-from-storage tier, warm promotion off - pod security throughout: runAsNonRoot uid 10001 (the image's user), seccomp RuntimeDefault, all capabilities dropped - overlays: minio-dev (self-contained on any local cluster) and aws (real S3; IRSA or access-key auth; optional secret so IRSA needs none) deploy/terraform/aws — the storage half: - private S3 bucket: public-access block, KMS encryption, TLS-only bucket policy, incomplete-multipart lifecycle cleanup, versioning deliberately off (the LSM manages object lifecycle; versioning would resurrect deleted fragments), force_destroy=false so destroy refuses on data - least-privilege IAM (object CRUD + list + multipart only), IRSA role with correctly-scoped OIDC trust, or opt-in IAM user + access key .github/workflows/docker.yml — publishes ghcr.io/runcaptain/compass (amd64+arm64) on release tags; the overlays reference it. Builder base rust:1.88-bookworm -> rust:1.88-trixie: bookworm's gcc-12 fails on the numkong dependency's ARM feature probes, making arm64 images impossible. Tested, not just written: minio-dev overlay deployed on a live k3s cluster — all tiers 1/1, PVC bound, 8/8 in-cluster checks through the Services (cross-tier read-your-writes writer->serving, role refusal, cold tier serving from the bucket), data survives serving-pod replacement via the PVC; terraform apply/re-apply/destroy exercised against an AWS API emulator (idempotent; destroy refuses while the bucket holds data); both image architectures built from the new base. Signed-off-by: Edgar Babajanyan --- .github/workflows/docker.yml | 56 ++++++ .gitignore | 7 + CHANGELOG.md | 7 + Dockerfile | 4 +- README.md | 16 ++ deploy/README.md | 93 +++++++++ deploy/kubernetes/base/common.yaml | 24 +++ deploy/kubernetes/base/kustomization.yaml | 10 + deploy/kubernetes/base/serving.yaml | 123 ++++++++++++ deploy/kubernetes/base/writer.yaml | 88 +++++++++ deploy/kubernetes/components/cold/cold.yaml | 89 +++++++++ .../components/cold/kustomization.yaml | 6 + .../overlays/aws/kustomization.yaml | 39 ++++ deploy/kubernetes/overlays/aws/namespace.yaml | 4 + .../overlays/minio-dev/kustomization.yaml | 61 ++++++ .../kubernetes/overlays/minio-dev/minio.yaml | 72 +++++++ .../overlays/minio-dev/namespace.yaml | 4 + deploy/terraform/aws/main.tf | 180 ++++++++++++++++++ deploy/terraform/aws/outputs.tf | 30 +++ deploy/terraform/aws/variables.tf | 56 ++++++ 20 files changed, 968 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/docker.yml create mode 100644 deploy/README.md create mode 100644 deploy/kubernetes/base/common.yaml create mode 100644 deploy/kubernetes/base/kustomization.yaml create mode 100644 deploy/kubernetes/base/serving.yaml create mode 100644 deploy/kubernetes/base/writer.yaml create mode 100644 deploy/kubernetes/components/cold/cold.yaml create mode 100644 deploy/kubernetes/components/cold/kustomization.yaml create mode 100644 deploy/kubernetes/overlays/aws/kustomization.yaml create mode 100644 deploy/kubernetes/overlays/aws/namespace.yaml create mode 100644 deploy/kubernetes/overlays/minio-dev/kustomization.yaml create mode 100644 deploy/kubernetes/overlays/minio-dev/minio.yaml create mode 100644 deploy/kubernetes/overlays/minio-dev/namespace.yaml create mode 100644 deploy/terraform/aws/main.tf create mode 100644 deploy/terraform/aws/outputs.tf create mode 100644 deploy/terraform/aws/variables.tf diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..10f05a6 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,56 @@ +name: Docker image + +# Publishes ghcr.io/runcaptain/compass on every release tag — the image the +# deploy/kubernetes overlays reference. Multi-arch (amd64 + arm64; the +# builder base is trixie specifically so arm64 compiles). Also runnable +# manually to publish an ad-hoc tag. + +on: + push: + tags: + - "v*" + workflow_dispatch: + +permissions: + contents: read + packages: write + +jobs: + publish: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=semver,pattern=v{{version}} + type=semver,pattern=v{{major}}.{{minor}} + type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=manual-${{ github.run_number }},enable=${{ github.event_name == 'workflow_dispatch' }} + + # The overlays assume the serverless topology, so the published image + # carries the object-storage backend. Local-first users typically build + # from source anyway (README quickstart). + - uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + build-args: | + CARGO_FEATURES=object-storage + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore index 251e0c1..8009221 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,10 @@ beir_data/ # Committed template — .env.* above must not swallow it !.env.example + +# Terraform (deploy/terraform) +.terraform/ +*.tfstate +*.tfstate.* +*.tfvars +.terraform.lock.hcl diff --git a/CHANGELOG.md b/CHANGELOG.md index fe3a9ae..3d7926d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- **One-command deployment** (`deploy/`): a kustomize tree for any Kubernetes cluster — serving StatefulSet with a PVC per replica (warm restarts), stateless writer Deployment, optional cold-serving tier, hardened pod security (non-root, seccomp, no capabilities), startup probes sized for cold rebuilds — plus Terraform for the AWS storage half (private KMS-encrypted S3 bucket with a TLS-only policy, least-privilege IAM, IRSA role or access key). A self-contained `minio-dev` overlay brings the whole serverless topology up on kind/k3s/minikube in one `kubectl apply -k`. Verified end-to-end on a live k3s cluster (all tiers serving, cross-tier read-your-writes, PVC persistence across pod replacement) and `terraform apply` verified against an AWS API emulator (idempotent re-apply; destroy refuses while data exists). +- **Published container images**: `ghcr.io/runcaptain/compass` (multi-arch amd64+arm64) on every release tag via `.github/workflows/docker.yml`. The Docker builder base moved to `rust:1.88-trixie` — bookworm's gcc-12 could not compile a dependency's ARM feature probes, which had made arm64 images impossible. + ## [0.4.0] - 2026-07-04 ### Added — serve-from-storage ("true serverless") diff --git a/Dockerfile b/Dockerfile index 71d18ca..001578d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,9 @@ # ── Stage 1: Build ──────────────────────────────────────────────────────────── # Pin builder toolchain so deploys are reproducible and a compromised # rust:latest tag can't silently land in our image. -FROM rust:1.88-bookworm AS builder +# trixie (gcc-14) rather than bookworm: bookworm's gcc-12 fails to build +# the numkong dependency's ARM feature probes, which blocked arm64 images. +FROM rust:1.88-trixie AS builder WORKDIR /app diff --git a/README.md b/README.md index 8d5041a..50871c5 100644 --- a/README.md +++ b/README.md @@ -419,6 +419,22 @@ docker build -t compass . docker run -p 4001:4001 -v ./data:/app/data compass ``` +## Deploy to Kubernetes / AWS (one command) + +[`deploy/`](deploy/README.md) ships the full serverless topology as code: a +kustomize tree (serving StatefulSet with a PVC per replica, stateless writer +Deployment, optional cold tier) plus Terraform for the AWS storage half +(private encrypted S3 bucket + least-privilege IAM, IRSA-ready). Try the whole +thing on any local cluster: + +```bash +kubectl apply -k deploy/kubernetes/overlays/minio-dev +``` + +Production: `terraform apply` in [`deploy/terraform/aws`](deploy/terraform/aws), +point [`deploy/kubernetes/overlays/aws`](deploy/kubernetes/overlays/aws) at the +bucket, `kubectl apply -k`. Details and operational notes: [deploy/README.md](deploy/README.md). + ## Object storage (S3 / GCS / Azure) By default Compass persists to local disk — zero config, no credentials. Optionally, it can persist to your own cloud object storage instead (a hard either/or, chosen at startup): diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..1eb278b --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,93 @@ +# Deploying Compass + +Three paths, in increasing order of ceremony. All of them are the same +binary; topology background lives in [docs/deployment.md](../docs/deployment.md). + +## 1. One machine (docker compose) + +```bash +docker compose up # local-first: data on the local volume, zero config +``` + +## 2. Kubernetes (any cluster) — the serverless topology + +`deploy/kubernetes` is a kustomize tree: + +``` +base/ serving StatefulSet (PVC per replica) + writer Deployment + + optional cold Deployment + services + config +overlays/minio-dev self-contained dev stack (in-cluster MinIO) — try it on + kind/k3s/minikube in one command +overlays/aws production: real S3, IRSA or access-key auth +``` + +Try the whole topology on a local cluster: + +```bash +kubectl apply -k deploy/kubernetes/overlays/minio-dev +kubectl -n compass-dev get pods # serving-0, writer, cold, minio +kubectl -n compass-dev port-forward svc/compass-read 4001:4001 +curl localhost:4001/health +``` + +Production on AWS: + +```bash +# 1. Storage half: bucket + least-privilege IAM +cd deploy/terraform/aws +terraform init && terraform apply -var bucket_name=my-compass-data +# EKS? add -var eks_oidc_provider_arn=… -var eks_oidc_provider_url=… +# no EKS? add -var create_access_key=true and create the Secret it hints at + +# 2. Compute half: point the overlay at your bucket + image, then +vi deploy/kubernetes/overlays/aws/kustomization.yaml # bucket, region, image, IRSA arn +kubectl apply -k deploy/kubernetes/overlays/aws +``` + +What you get: + +| workload | kind | storage | scale advice | +|---|---|---|---| +| `compass-serving` | StatefulSet | PVC per replica (warm restarts = seconds) | scale for read QPS; each replica converges independently | +| `compass-writer` | Deployment | none (stateless) | scale for ingest; safe to kill any time | +| `compass-cold` | Deployment (optional) | none | scale-to-many for bursty semantic reads on rarely-touched collections | + +Traffic contract: **reads → `compass-read`, writes → `compass-write`**, +optional cold reads → `compass-cold`. Writers refuse reads by design; +serving nodes accept both but you keep clean scaling curves by splitting. + +## 3. Terraform (AWS storage half) + +`deploy/terraform/aws` provisions the bucket (private, encrypted, +lifecycle-managed) and least-privilege IAM — IRSA role for EKS, or an IAM +user + access key for anything else. It deliberately does NOT create a +cluster; bring any Kubernetes (or run the containers on VMs). + +## Operational notes (read before production) + +- **The bucket is the database.** PVCs are a warm cache — losing one costs a + rebuild, never data. Bucket deletion is data loss; `force_destroy` stays + false for a reason. +- **Consistency**: writes through writers are durable immediately and + visible on serving nodes within `COMPASS_REFRESH_INTERVAL` (default 5s). + Pass a write's `seq` as `min_seq` on search for read-your-writes; cold + pods have read-your-writes by construction. +- **Version skew**: never run pre-v0.4 and v0.4+ writers against one bucket. + Roll writers first, then serving nodes. +- **Auth**: the API is unauthenticated until you set `COMPASS_API_KEY`. The + manifests already envFrom the `compass-aws` secret, so add the key there: + `kubectl -n compass create secret generic compass-aws --from-literal=COMPASS_API_KEY=… [--from-literal=AWS_…]`. + `/health` and `/metrics` stay unauthenticated by design; keep them + cluster-internal (no NetworkPolicy ships here — add one if your cluster + doesn't default-deny). +- **PVC lifecycle**: `kubectl delete -k …` removes the pods but RETAINS the + StatefulSet PVCs (Kubernetes default) — re-applying reuses the warm cache. + Delete PVCs explicitly to reclaim disk; that costs a rebuild, never data. +- **Single-replica updates**: with `replicas: 1`, a rolling update has a + brief read-downtime window while the pod restarts (writes keep flowing via + writers). Run ≥2 serving replicas if reads must never blip. +- **Sizing**: serving-node RAM tracks attached collections (bound it with + `COMPASS_LAZY_ATTACH` + `COMPASS_MAX_ATTACHED`); PVC size tracks the same + data as the bucket per attached collection. Cold pods run in ~tens of MiB. +- **Upgrades**: StatefulSet updates roll one pod at a time; readiness gating + keeps traffic off a pod until its indexes serve. Writers roll freely. diff --git a/deploy/kubernetes/base/common.yaml b/deploy/kubernetes/base/common.yaml new file mode 100644 index 0000000..7cc295c --- /dev/null +++ b/deploy/kubernetes/base/common.yaml @@ -0,0 +1,24 @@ +# Shared plumbing: namespace-agnostic service account + engine config. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: compass + # On EKS with IRSA, the aws overlay annotates this with the role ARN + # from the Terraform output — no long-lived keys in the cluster. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: compass-config +data: + PORT: "4001" + RUST_LOG: "compass=info" + # The bucket. Overlays set this (s3://…, gs://…, az://…). Empty = the + # engine falls back to LOCAL disk mode — a bare `apply -k base` still + # boots, it just isn't the serverless topology. + COMPASS_STORAGE: "" + # Serving nodes converge on other nodes' writes every N seconds. + COMPASS_REFRESH_INTERVAL: "5" + # Uncomment to bound serving RAM to the hot collection set: + # COMPASS_LAZY_ATTACH: "true" + # COMPASS_MAX_ATTACHED: "64" diff --git a/deploy/kubernetes/base/kustomization.yaml b/deploy/kubernetes/base/kustomization.yaml new file mode 100644 index 0000000..2789180 --- /dev/null +++ b/deploy/kubernetes/base/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - common.yaml + - serving.yaml + - writer.yaml +labels: + - pairs: + app.kubernetes.io/part-of: compass + includeSelectors: false diff --git a/deploy/kubernetes/base/serving.yaml b/deploy/kubernetes/base/serving.yaml new file mode 100644 index 0000000..ea09eaf --- /dev/null +++ b/deploy/kubernetes/base/serving.yaml @@ -0,0 +1,123 @@ +# Serving nodes: full local indexes, fast reads, background convergence. +# +# StatefulSet — NOT a Deployment — because each replica needs its OWN +# persistent volume: a warm disk turns restarts into a seconds-long delta +# catch-up (applied_seq) instead of a full rebuild from the bucket. Two +# replicas sharing one PVC would corrupt each other's indexes. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: compass-serving +spec: + serviceName: compass-serving-headless + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: compass + app.kubernetes.io/component: serving + template: + metadata: + labels: + app.kubernetes.io/name: compass + app.kubernetes.io/component: serving + spec: + serviceAccountName: compass + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + # Skip the recursive chown when the PVC root already has the right + # group — a full walk of a big warm cache would stall pod starts. + fsGroupChangePolicy: OnRootMismatch + containers: + - name: compass + image: compass:latest # kustomize overlays pin the real image + ports: + - containerPort: 4001 + name: http + envFrom: + - configMapRef: + name: compass-config + - secretRef: + name: compass-aws + optional: true # IRSA mode needs no secret + env: + - name: DATA_DIR + value: /data + volumeMounts: + - name: data + mountPath: /data + # Boot order matters: the engine loads/rebuilds every collection + # BEFORE binding the port, so a node recovering a big bucket + # answers nothing for minutes. The startupProbe owns that window + # (up to 60 min) — liveness only takes over once the port is up. + # For very large buckets, prefer COMPASS_LAZY_ATTACH=true, which + # makes boot instant and this probe trivially green. + startupProbe: + httpGet: + path: /health + port: http + periodSeconds: 15 + failureThreshold: 240 + readinessProbe: + httpGet: + path: /health + port: http + periodSeconds: 5 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /health + port: http + periodSeconds: 20 + failureThreshold: 6 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + memory: 4Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false # tantivy/redb write under /data only, but keep tmp writable + capabilities: + drop: ["ALL"] + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 20Gi +--- +# Headless service the StatefulSet requires (stable per-pod DNS). +apiVersion: v1 +kind: Service +metadata: + name: compass-serving-headless +spec: + clusterIP: None + selector: + app.kubernetes.io/name: compass + app.kubernetes.io/component: serving + ports: + - port: 4001 + targetPort: http + name: http +--- +# Read endpoint: load-balances across ready serving pods. +apiVersion: v1 +kind: Service +metadata: + name: compass-read +spec: + selector: + app.kubernetes.io/name: compass + app.kubernetes.io/component: serving + ports: + - port: 4001 + targetPort: http + name: http diff --git a/deploy/kubernetes/base/writer.yaml b/deploy/kubernetes/base/writer.yaml new file mode 100644 index 0000000..9863d0f --- /dev/null +++ b/deploy/kubernetes/base/writer.yaml @@ -0,0 +1,88 @@ +# Writer nodes: stateless, append-only ingest. Durable immediately; +# searchable on serving nodes within the refresh interval. +# +# Plain Deployment with an emptyDir scratch dir — writers hold no state +# worth keeping (ids come from the bucket's CAS allocator), so they scale +# horizontally and die freely. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: compass-writer +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: compass + app.kubernetes.io/component: writer + template: + metadata: + labels: + app.kubernetes.io/name: compass + app.kubernetes.io/component: writer + spec: + serviceAccountName: compass + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: compass + image: compass:latest + ports: + - containerPort: 4001 + name: http + envFrom: + - configMapRef: + name: compass-config + - secretRef: + name: compass-aws + optional: true + env: + - name: DATA_DIR + value: /data + - name: COMPASS_ROLE + value: writer + volumeMounts: + - name: scratch + mountPath: /data + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 2 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 5 + periodSeconds: 20 + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + memory: 1Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumes: + - name: scratch + emptyDir: {} +--- +# Write endpoint: point ingest/delete traffic here. +apiVersion: v1 +kind: Service +metadata: + name: compass-write +spec: + selector: + app.kubernetes.io/name: compass + app.kubernetes.io/component: writer + ports: + - port: 4001 + targetPort: http + name: http diff --git a/deploy/kubernetes/components/cold/cold.yaml b/deploy/kubernetes/components/cold/cold.yaml new file mode 100644 index 0000000..daa679f --- /dev/null +++ b/deploy/kubernetes/components/cold/cold.yaml @@ -0,0 +1,89 @@ +# OPTIONAL cold-serving tier: answers semantic queries on collections it has +# never attached, straight from object-storage range reads. Boots in <1s at +# ~30MiB; scale-to-many for bursty read fleets. Not part of the base — an +# overlay opts in with `resources: [../../components/cold]`. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: compass-cold +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: compass + app.kubernetes.io/component: cold + template: + metadata: + labels: + app.kubernetes.io/name: compass + app.kubernetes.io/component: cold + spec: + serviceAccountName: compass + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: compass + image: compass:latest + ports: + - containerPort: 4001 + name: http + envFrom: + - configMapRef: + name: compass-config + - secretRef: + name: compass-aws + optional: true + env: + - name: DATA_DIR + value: /data + - name: COMPASS_COLD_SERVE + value: "true" + # Cold pods stay cold: warm promotion would re-grow local state + # on a tier sized for none. Route hot tenants to serving pods. + - name: COMPASS_WARM_AFTER + value: "0" + volumeMounts: + - name: scratch + mountPath: /data + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 2 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 5 + periodSeconds: 20 + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + memory: 1Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumes: + - name: scratch + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: compass-cold +spec: + selector: + app.kubernetes.io/name: compass + app.kubernetes.io/component: cold + ports: + - port: 4001 + targetPort: http + name: http diff --git a/deploy/kubernetes/components/cold/kustomization.yaml b/deploy/kubernetes/components/cold/kustomization.yaml new file mode 100644 index 0000000..2c511a7 --- /dev/null +++ b/deploy/kubernetes/components/cold/kustomization.yaml @@ -0,0 +1,6 @@ +# Optional cold-serving tier — include from an overlay with: +# resources: [..., ../../components/cold] +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - cold.yaml diff --git a/deploy/kubernetes/overlays/aws/kustomization.yaml b/deploy/kubernetes/overlays/aws/kustomization.yaml new file mode 100644 index 0000000..c74f24e --- /dev/null +++ b/deploy/kubernetes/overlays/aws/kustomization.yaml @@ -0,0 +1,39 @@ +# Production overlay for AWS S3. Two auth modes: +# IRSA (recommended): set the service-account annotation below and skip +# the secret entirely. +# Access key: create the secret Terraform hints at +# (kubectl -n compass create secret generic compass-aws …) — the base +# mounts it as optional, so IRSA deployments simply omit it. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: compass +resources: + - namespace.yaml + - ../../base + - ../../components/cold # optional cold tier; delete this line to skip + +images: + - name: compass + # ghcr.io/runcaptain/compass is published by .github/workflows/docker.yml + # on every release tag (multi-arch). Swap in your own registry if you + # build custom images. + newName: ghcr.io/runcaptain/compass + newTag: v0.4.1 + +patches: + - patch: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: compass-config + data: + COMPASS_STORAGE: "s3://CHANGE-ME-your-bucket" # terraform output compass_storage_url + AWS_REGION: "us-east-1" + # IRSA mode: uncomment and paste the terraform `irsa_role_arn` output. + # - patch: |- + # apiVersion: v1 + # kind: ServiceAccount + # metadata: + # name: compass + # annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/compass-irsa diff --git a/deploy/kubernetes/overlays/aws/namespace.yaml b/deploy/kubernetes/overlays/aws/namespace.yaml new file mode 100644 index 0000000..fae496b --- /dev/null +++ b/deploy/kubernetes/overlays/aws/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: compass diff --git a/deploy/kubernetes/overlays/minio-dev/kustomization.yaml b/deploy/kubernetes/overlays/minio-dev/kustomization.yaml new file mode 100644 index 0000000..89738d5 --- /dev/null +++ b/deploy/kubernetes/overlays/minio-dev/kustomization.yaml @@ -0,0 +1,61 @@ +# Self-contained dev/test overlay: in-cluster MinIO stands in for S3. +# One command gives you the full serverless topology on any cluster +# (kind/k3s/minikube included): +# kubectl apply -k deploy/kubernetes/overlays/minio-dev +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: compass-dev +resources: + - namespace.yaml + - minio.yaml + - ../../base + - ../../components/cold + +images: + - name: compass + newName: compass # expects a locally imported image in dev clusters + newTag: latest + +secretGenerator: + - name: compass-aws + literals: + - AWS_ACCESS_KEY_ID=minioadmin + - AWS_SECRET_ACCESS_KEY=minioadmin + options: + disableNameSuffixHash: true + +patches: + # The dev image is imported into the cluster (kind load / ctr images + # import); :latest defaults to pullPolicy Always, which would try (and + # fail) to pull from a registry. Never = use the imported image only. + - target: + kind: StatefulSet + name: compass-serving + patch: |- + - op: add + path: /spec/template/spec/containers/0/imagePullPolicy + value: Never + - target: + kind: Deployment + name: compass-writer + patch: |- + - op: add + path: /spec/template/spec/containers/0/imagePullPolicy + value: Never + - target: + kind: Deployment + name: compass-cold + patch: |- + - op: add + path: /spec/template/spec/containers/0/imagePullPolicy + value: Never + - patch: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: compass-config + data: + COMPASS_STORAGE: "s3://compass-data" + COMPASS_S3_ENDPOINT: "http://minio:9000" + COMPASS_S3_ALLOW_HTTP: "true" + AWS_REGION: "us-east-1" diff --git a/deploy/kubernetes/overlays/minio-dev/minio.yaml b/deploy/kubernetes/overlays/minio-dev/minio.yaml new file mode 100644 index 0000000..ce6815f --- /dev/null +++ b/deploy/kubernetes/overlays/minio-dev/minio.yaml @@ -0,0 +1,72 @@ +# In-cluster MinIO for the dev overlay. NOT for production — use real S3/GCS +# (the aws overlay) and let the cloud own durability. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: minio + template: + metadata: + labels: + app.kubernetes.io/name: minio + spec: + containers: + - name: minio + image: minio/minio:latest + args: ["server", "/data"] + env: + - name: MINIO_ROOT_USER + value: minioadmin + - name: MINIO_ROOT_PASSWORD + value: minioadmin + ports: + - containerPort: 9000 + name: s3 + readinessProbe: + httpGet: + path: /minio/health/live + port: s3 + initialDelaySeconds: 2 + periodSeconds: 3 + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: minio +spec: + selector: + app.kubernetes.io/name: minio + ports: + - port: 9000 + targetPort: s3 + name: s3 +--- +# One-shot job: create the bucket Compass points at. +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-mkbucket +spec: + backoffLimit: 6 + template: + spec: + restartPolicy: OnFailure + containers: + - name: mc + image: minio/mc:latest + command: + - sh + - -c + - | + until mc alias set local http://minio:9000 minioadmin minioadmin; do sleep 2; done + mc mb -p local/compass-data diff --git a/deploy/kubernetes/overlays/minio-dev/namespace.yaml b/deploy/kubernetes/overlays/minio-dev/namespace.yaml new file mode 100644 index 0000000..49ff3f7 --- /dev/null +++ b/deploy/kubernetes/overlays/minio-dev/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: compass-dev diff --git a/deploy/terraform/aws/main.tf b/deploy/terraform/aws/main.tf new file mode 100644 index 0000000..4c4fee1 --- /dev/null +++ b/deploy/terraform/aws/main.tf @@ -0,0 +1,180 @@ +# Compass on AWS — the storage half of a deployment. +# +# Provisions exactly what Compass needs from AWS and nothing else: +# - an S3 bucket (the database: source of truth for every collection) +# - a least-privilege IAM policy scoped to that bucket +# - EITHER an IRSA role for EKS service accounts (recommended) +# OR an IAM user + access key (for non-EKS clusters / VMs) +# +# The compute half lives in ../../kubernetes (any cluster) or plain +# containers — see deploy/README.md. Nothing here creates a cluster: +# bring your own EKS/K8s, or run the containers however you like. + +terraform { + required_version = ">= 1.5" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } +} + +# ── The bucket: this IS the database ───────────────────────────────────────── + +resource "aws_s3_bucket" "compass" { + bucket = var.bucket_name + + # The bucket holds every collection; force_destroy=false means + # `terraform destroy` refuses while data exists. Flip deliberately. + force_destroy = var.force_destroy +} + +resource "aws_s3_bucket_public_access_block" "compass" { + bucket = aws_s3_bucket.compass.id + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "compass" { + bucket = aws_s3_bucket.compass.id + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "aws:kms" + # null = the account's default aws/s3 KMS key; set to use your own CMK. + kms_master_key_id = var.kms_key_arn + } + bucket_key_enabled = true + } +} + +# Compass manages object lifecycle itself (LSM compaction + deferred GC). +# Versioning would resurrect deleted WAL fragments and double storage cost — +# deliberately OFF. Point-in-time recovery = S3 replication if you need it. + +resource "aws_s3_bucket_lifecycle_configuration" "compass" { + bucket = aws_s3_bucket.compass.id + rule { + id = "abort-incomplete-multipart" + status = "Enabled" + filter {} + # Crashed multipart uploads (large segment writes) otherwise bill forever. + abort_incomplete_multipart_upload { + days_after_initiation = 7 + } + } +} + +# Deny any non-TLS access outright — defense in depth for a bucket that +# holds every collection. +resource "aws_s3_bucket_policy" "tls_only" { + bucket = aws_s3_bucket.compass.id + policy = data.aws_iam_policy_document.tls_only.json + + # The public-access block must land first or the policy PUT can race it. + depends_on = [aws_s3_bucket_public_access_block.compass] +} + +data "aws_iam_policy_document" "tls_only" { + statement { + sid = "DenyInsecureTransport" + effect = "Deny" + actions = ["s3:*"] + principals { + type = "*" + identifiers = ["*"] + } + resources = [aws_s3_bucket.compass.arn, "${aws_s3_bucket.compass.arn}/*"] + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + } +} + +# ── Least-privilege access policy ──────────────────────────────────────────── +# Compass needs: read/write/delete objects, list the bucket, multipart +# uploads. It does NOT need bucket administration, ACLs, or anything +# account-wide — and this policy grants none of that. + +data "aws_iam_policy_document" "compass" { + statement { + sid = "CompassObjects" + actions = ["s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts"] + resources = ["${aws_s3_bucket.compass.arn}/*"] + } + statement { + sid = "CompassList" + actions = ["s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:GetBucketLocation"] + resources = [aws_s3_bucket.compass.arn] + } +} + +resource "aws_iam_policy" "compass" { + name = "${var.name_prefix}-s3" + policy = data.aws_iam_policy_document.compass.json +} + +# ── Mode A (recommended): IRSA role for EKS ───────────────────────────────── +# Pods assume this role via their service account — no long-lived keys +# anywhere. Set eks_oidc_provider_arn + eks_oidc_provider_url to enable. + +locals { + irsa_enabled = var.eks_oidc_provider_arn != null +} + +data "aws_iam_policy_document" "irsa_trust" { + count = local.irsa_enabled ? 1 : 0 + statement { + actions = ["sts:AssumeRoleWithWebIdentity"] + principals { + type = "Federated" + identifiers = [var.eks_oidc_provider_arn] + } + condition { + test = "StringEquals" + variable = "${trimprefix(var.eks_oidc_provider_url, "https://")}:sub" + values = ["system:serviceaccount:${var.k8s_namespace}:${var.k8s_service_account}"] + } + condition { + test = "StringEquals" + variable = "${trimprefix(var.eks_oidc_provider_url, "https://")}:aud" + values = ["sts.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "compass_irsa" { + count = local.irsa_enabled ? 1 : 0 + name = "${var.name_prefix}-irsa" + assume_role_policy = data.aws_iam_policy_document.irsa_trust[0].json +} + +resource "aws_iam_role_policy_attachment" "compass_irsa" { + count = local.irsa_enabled ? 1 : 0 + role = aws_iam_role.compass_irsa[0].name + policy_arn = aws_iam_policy.compass.arn +} + +# ── Mode B: IAM user + access key (non-EKS clusters, VMs, dev) ────────────── +# Long-lived credentials; rotate them, keep them in a K8s Secret, and prefer +# IRSA when you're on EKS. Off by default. + +resource "aws_iam_user" "compass" { + count = var.create_access_key ? 1 : 0 + name = "${var.name_prefix}-svc" +} + +resource "aws_iam_user_policy_attachment" "compass" { + count = var.create_access_key ? 1 : 0 + user = aws_iam_user.compass[0].name + policy_arn = aws_iam_policy.compass.arn +} + +resource "aws_iam_access_key" "compass" { + count = var.create_access_key ? 1 : 0 + user = aws_iam_user.compass[0].name +} diff --git a/deploy/terraform/aws/outputs.tf b/deploy/terraform/aws/outputs.tf new file mode 100644 index 0000000..f649976 --- /dev/null +++ b/deploy/terraform/aws/outputs.tf @@ -0,0 +1,30 @@ +output "bucket_name" { + value = aws_s3_bucket.compass.bucket + description = "The bucket that holds every collection." +} + +output "compass_storage_url" { + value = "s3://${aws_s3_bucket.compass.bucket}" + description = "Value for the COMPASS_STORAGE environment variable." +} + +output "irsa_role_arn" { + value = local.irsa_enabled ? aws_iam_role.compass_irsa[0].arn : null + description = "Annotate the Compass service account with this (eks.amazonaws.com/role-arn) when using IRSA." +} + +output "access_key_id" { + value = var.create_access_key ? aws_iam_access_key.compass[0].id : null + description = "AWS_ACCESS_KEY_ID for mode B. Store in a Kubernetes Secret." +} + +output "secret_access_key" { + value = var.create_access_key ? aws_iam_access_key.compass[0].secret : null + sensitive = true + description = "AWS_SECRET_ACCESS_KEY for mode B. `terraform output -raw secret_access_key`." +} + +output "kubernetes_secret_hint" { + value = var.create_access_key ? "kubectl -n compass create secret generic compass-aws --from-literal=AWS_ACCESS_KEY_ID=$(terraform output -raw access_key_id) --from-literal=AWS_SECRET_ACCESS_KEY=$(terraform output -raw secret_access_key)" : "IRSA mode: no secret needed — annotate the service account with irsa_role_arn." + description = "Next step after apply." +} diff --git a/deploy/terraform/aws/variables.tf b/deploy/terraform/aws/variables.tf new file mode 100644 index 0000000..c33f063 --- /dev/null +++ b/deploy/terraform/aws/variables.tf @@ -0,0 +1,56 @@ +variable "bucket_name" { + description = "Globally-unique S3 bucket name. This bucket IS the database." + type = string +} + +variable "name_prefix" { + description = "Prefix for IAM resources (policy, role, user)." + type = string + default = "compass" +} + +variable "force_destroy" { + description = "Allow `terraform destroy` to delete a NON-EMPTY bucket. Leave false: flipping it and destroying erases every collection." + type = bool + default = false +} + +variable "kms_key_arn" { + description = "Optional customer-managed KMS key for bucket encryption. null = AWS-managed aws/s3 key." + type = string + default = null +} + +# ── Mode A: IRSA (EKS) ────────────────────────────────────────────────────── + +variable "eks_oidc_provider_arn" { + description = "ARN of the cluster's OIDC provider (aws_iam_openid_connect_provider). Set together with eks_oidc_provider_url to mint an IRSA role." + type = string + default = null +} + +variable "eks_oidc_provider_url" { + description = "URL of the cluster's OIDC provider (with or without https://)." + type = string + default = null +} + +variable "k8s_namespace" { + description = "Namespace of the Compass service account (IRSA trust condition)." + type = string + default = "compass" +} + +variable "k8s_service_account" { + description = "Name of the Compass service account (IRSA trust condition)." + type = string + default = "compass" +} + +# ── Mode B: access key ────────────────────────────────────────────────────── + +variable "create_access_key" { + description = "Create an IAM user + long-lived access key instead of (or in addition to) IRSA. For non-EKS clusters and VMs." + type = bool + default = false +} From abc26401065e21a8cdf2b3bc0c33e7c7975985d3 Mon Sep 17 00:00:00 2001 From: Edgar Babajanyan Date: Mon, 6 Jul 2026 13:42:38 -0700 Subject: [PATCH 2/2] Update cxx to 1.0.196 (RUSTSEC-2026-0202) The advisory landed 2026-07-05 and fails cargo audit on every PR. cxx is a transitive dependency (usearch bindings); the unsound let_cxx_string! macro is not used anywhere in this workspace, but the clean fix is the patched version rather than an audit ignore. Suites unaffected. Signed-off-by: Edgar Babajanyan --- Cargo.lock | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0485e67..08dacd9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -643,9 +643,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.194" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747d8437319e3a2f43d93b341c137927ca70c0f5dabeea7a005a73665e247c7e" +checksum = "ac6f8908ed0826ab0aec7c8358a4de3a9b26c5ed391a5dc5135765d2fd1aa8fb" dependencies = [ "cc", "cxx-build", @@ -658,9 +658,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.194" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0f4697d190a142477b16aef7da8a99bfdc41e7e8b1687583c0d23a79c7afc1e" +checksum = "6d8ae25c0ce72ac21a77b5deec4f49b452238ef97072f697dd6fd752b1355ecd" dependencies = [ "cc", "codespan-reporting", @@ -673,9 +673,9 @@ dependencies = [ [[package]] name = "cxxbridge-cmd" -version = "1.0.194" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0956799fa8678d4c50eed028f2de1c0552ae183c76e976cf7ca8c4e36a7c328" +checksum = "27d53791812143bd27f74ab6055f22e875f04c59bbef0ca03d714ebec6f6f484" dependencies = [ "clap", "codespan-reporting", @@ -687,15 +687,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.194" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23384a836ab4f0ad98ace7e3955ad2de39de42378ab487dc28d3990392cb283a" +checksum = "fd557619f7dc2252bf7373f6bec5600ce60a2b477e5b3b84eb4e074bb297795e" [[package]] name = "cxxbridge-macro" -version = "1.0.194" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6acc6b5822b9526adfb4fc377b67128fdd60aac757cc4a741a6278603f763cf" +checksum = "7fe465fc5b9d0231ea141c4fae714a08f4f907855e1a7ca10cc90ab6c8b12ece" dependencies = [ "indexmap", "proc-macro2", @@ -889,7 +889,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2029,7 +2029,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2782,7 +2782,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3317,7 +3317,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3987,7 +3987,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]]