From 9c509557bdbcb778c7908470b353212156f4a348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Plaza=20Sisqu=C3=A9s?= Date: Tue, 1 Sep 2026 11:07:39 +0200 Subject: [PATCH] feat(keycloak): add shared Keycloak instance for account-api and future services account-api's Keycloak used to live in its own docker-compose.yml because it had exactly one consumer; now that more than one service needs it, it moves here alongside the other shared infra (Postgres, Kafka, Redis, OTel). Realms are imported via the Admin REST API by a one-shot keycloak-realm-import job, not Keycloak's own --import-realm startup flag: that flag reliably crashes Keycloak 26.0 on boot with "ERROR: Session not bound to a realm" right after logging the realm import as successful (reproduced repeatedly while adding this service; see upstream keycloak/keycloak#33637 and #34673, unresolved as of image 26.0.8). --- .env.example | 5 ++ README.md | 85 +++++++++++++++++-- docker-compose.yml | 33 +++++++ docker/keycloak/import-realms.sh | 55 ++++++++++++ docker/keycloak/realms/account-api-realm.json | 24 ++++++ 5 files changed, 197 insertions(+), 5 deletions(-) create mode 100755 docker/keycloak/import-realms.sh create mode 100644 docker/keycloak/realms/account-api-realm.json diff --git a/.env.example b/.env.example index 6afda0e..fd15c84 100644 --- a/.env.example +++ b/.env.example @@ -20,6 +20,11 @@ MONGO_EXPRESS_USERNAME=admin MONGO_EXPRESS_PASSWORD=devpassword MONGO_EXPRESS_PORT=8081 +# --- Keycloak --- +KEYCLOAK_ADMIN_USERNAME=admin +KEYCLOAK_ADMIN_PASSWORD=admin +KEYCLOAK_PORT=8084 + # --- Kafka --- KAFKA_PORT=9092 KAFKA_UI_PORT=8080 diff --git a/README.md b/README.md index 9dae38b..b63817d 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,9 @@ points at it. - Docker and Docker Compose v2 (`docker compose`, not the legacy `docker-compose`). -- Ports 5432, 6379, 8080, 9092, 9090, 16686, 4317, 4318 and 8889 free on - your machine (see [Ports used](#ports-used) below for how to change any of - them). +- Ports 5432, 6379, 5540, 27017, 8081, 8084, 8080, 9092, 9090, 16686, 4317, + 4318 and 8889 free on your machine (see [Ports used](#ports-used) below + for how to change any of them). ## Getting started @@ -45,8 +45,9 @@ docker compose down ``` Stop the stack **and delete all data** (Postgres, Redis, RedisInsight, -MongoDB, Prometheus — see the [Kafka persistence](#kafka-persistence) note -below for why Kafka isn't listed): +MongoDB, Prometheus — see the [Kafka persistence](#kafka-persistence) +note below for why Kafka isn't listed, and the [Keycloak](#keycloak) section +for why it has no data volume to delete in the first place): ```bash docker compose down -v @@ -65,6 +66,8 @@ If you don't create a `.env` file, the defaults baked into | RedisInsight | `redis/redisinsight:2.60` | Web UI to browse/inspect Redis keys | | MongoDB | `mongo:7.0.15` | Shared instance, root auth (kept for future use — no service uses it yet) | | Mongo Express | `mongo-express:1.0.2-20` | Web UI to browse/inspect MongoDB collections | +| Keycloak | `quay.io/keycloak/keycloak:26.0` | Shared identity provider (OpenID Connect / OAuth2) | +| Keycloak realm import | `curlimages/curl:8.11.0` | One-shot job, imports `docker/keycloak/realms/*.json` | | Kafka | `apache/kafka:3.8.0` | Single-broker cluster, KRaft mode (no Zookeeper) | | Kafka UI | `ghcr.io/kafbat/kafka-ui:v1.0.0` | Web UI to inspect topics/messages | | Jaeger | `jaegertracing/all-in-one:1.60` | Trace collector + UI | @@ -103,6 +106,37 @@ authentication database/user per service. Data persists in `local-dev-stack-mongo-data`. Browse it via **Mongo Express** at http://localhost:8081 (basic-auth login, default `admin` / `devpassword`). +### Keycloak + +A single shared Keycloak instance, run with `start-dev` (dev mode — not +hardened for production, fine for local use). Admin console at +http://localhost:8084, login with `KEYCLOAK_ADMIN_USERNAME` / +`KEYCLOAK_ADMIN_PASSWORD` (both default to `admin`). + +**No persistent data volume, by design** — Keycloak runs with ephemeral +in-memory/H2 storage, so any changes made through the admin console are +lost on `docker compose down` / container recreation (realms get +re-imported on every `docker compose up -d` anyway — see below). + +**Realms are imported via the Admin REST API, not Keycloak's own +`--import-realm` flag.** A one-shot `keycloak-realm-import` service +(`curlimages/curl`, see `docker/keycloak/import-realms.sh`) waits for +Keycloak to be ready, then POSTs every file in `docker/keycloak/realms/` to +`/admin/realms`, and exits. This is deliberate, not a style preference: +`start-dev --import-realm` reliably crashes Keycloak 26.0 on boot with +`ERROR: Session not bound to a realm` right after logging +`Realm '' imported` — reproduced repeatedly while adding this +service (see upstream +[keycloak/keycloak#33637](https://github.com/keycloak/keycloak/issues/33637) +and +[#34673](https://github.com/keycloak/keycloak/issues/34673), both +unresolved as of image `26.0.8`). Don't switch back to `--import-realm` +without confirming that bug is actually fixed upstream. + +See [Adding a new service's realm](#adding-a-new-services-realm) for how a +service registers its own realm/client here, mirroring how +`docker/postgres/init-db.sh` centralizes one database per service. + ### Kafka A single-broker Kafka cluster running in **KRaft mode** (no Zookeeper), @@ -181,6 +215,41 @@ array. To add a new service's database: -c "CREATE DATABASE billing_service_db;" ``` +## Adding a new service's realm + +Realm exports are dropped as individual JSON files in +`docker/keycloak/realms/` — the `keycloak-realm-import` job (see +[Keycloak](#keycloak)) `POST`s every file in that directory to +`/admin/realms` once Keycloak is up. To register a new service: + +1. Export (or hand-write) a realm JSON containing the service's clients and + roles, e.g. `docker/keycloak/realms/billing-service-realm.json`. Don't + include a top-level `users` block that assigns `clientRoles` on a + service-account user (e.g. `realm-management` roles for a client with + `serviceAccountsEnabled: true`) — that shape crashed Keycloak 26.0 when + imported via the old `--import-realm` flag (reproduced while adding + `account-api-realm.json` here, see [Keycloak](#keycloak)); grant those + role mappings once after import instead, either in the admin console + (Users → service-account-`` → Role mapping) or via the Admin + REST API. +2. Re-run the import job: + + ```bash + docker compose up -d keycloak-realm-import + ``` + + The import only **creates** realms — it doesn't update ones that already + exist (`POST /admin/realms` returns `409 Conflict`, which the script + treats as "nothing to do", not an error). So this picks up a **new** + realm file immediately, but editing an **existing** realm file and + re-running the job does nothing. To apply an edit to an existing realm, + either make the change by hand in the admin console, or delete the realm + first (admin console, or `docker compose down -v && docker compose up -d` + to wipe every service's data and start clean) before re-running the job. + +Keep secrets in these files at local-dev-only placeholder values (same rule +as everything else in this repo — see [Credentials](#credentials)). + ## Pointing a nestjs-template service at this stack Your service needs to be on the same Docker network @@ -228,6 +297,11 @@ MONGO_PORT=27017 MONGO_USERNAME=devuser MONGO_PASSWORD=devpassword +# Keycloak — issuer URL uses the internal hostname; admin API only if needed +KEYCLOAK_ISSUER_URL=http://keycloak:8080/realms/ +KEYCLOAK_CLIENT_ID= +KEYCLOAK_CLIENT_SECRET= + # OpenTelemetry — traces/metrics go to the collector, never straight to Jaeger OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 OTEL_SERVICE_NAME= @@ -248,6 +322,7 @@ hostnames — e.g. `KAFKA_BROKERS=localhost:9092`, | RedisInsight | 5540 | Web UI for Redis | | MongoDB | 27017 | Database connections | | Mongo Express | 8081 | Web UI for MongoDB | +| Keycloak | 8084 | Admin console + OpenID Connect / OAuth2 API | | Kafka | 9092 | Broker (`PLAINTEXT_HOST` listener) | | Kafka UI | 8080 | Web UI to browse topics/messages | | Jaeger UI | 16686 | Web UI for traces | diff --git a/docker-compose.yml b/docker-compose.yml index cc81063..a4911e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,6 +89,39 @@ services: networks: - local-dev-stack-net + keycloak: + image: quay.io/keycloak/keycloak:26.0 + container_name: local-dev-stack-keycloak + restart: unless-stopped + command: ["start-dev"] + environment: + KC_BOOTSTRAP_ADMIN_USERNAME: ${KEYCLOAK_ADMIN_USERNAME:-admin} + KC_BOOTSTRAP_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD:-admin} + ports: + - "${KEYCLOAK_PORT:-8084}:8080" + networks: + - local-dev-stack-net + + # One-shot job: imports docker/keycloak/realms/*.json via the Admin REST + # API once Keycloak is up, then exits. Not run as Keycloak's own + # `--import-realm` startup flag — see docker/keycloak/import-realms.sh for + # why (a reproducible Keycloak 26.0 startup crash). No `restart:` — it's + # meant to run once per `docker compose up`, not loop. + keycloak-realm-import: + image: curlimages/curl:8.11.0 + container_name: local-dev-stack-keycloak-realm-import + entrypoint: ["/bin/sh", "/import-realms.sh"] + environment: + KEYCLOAK_ADMIN_USERNAME: ${KEYCLOAK_ADMIN_USERNAME:-admin} + KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD:-admin} + volumes: + - ./docker/keycloak/realms:/realms:ro + - ./docker/keycloak/import-realms.sh:/import-realms.sh:ro + depends_on: + - keycloak + networks: + - local-dev-stack-net + kafka: image: apache/kafka:3.8.0 container_name: local-dev-stack-kafka diff --git a/docker/keycloak/import-realms.sh b/docker/keycloak/import-realms.sh new file mode 100755 index 0000000..60c05af --- /dev/null +++ b/docker/keycloak/import-realms.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# Imports every docker/keycloak/realms/*.json into the running Keycloak via +# the Admin REST API, instead of Keycloak's own `--import-realm` startup +# flag. `--import-realm` is unusable on Keycloak 26.0: it reliably crashes +# the server on boot with "ERROR: Session not bound to a realm" right after +# logging "Realm '' imported" (reproduced repeatedly while adding this +# service — see upstream keycloak/keycloak#33637 and #34673, unresolved as +# of 26.0.8). The REST API path below hits the same import endpoint Keycloak +# uses internally, without that startup-time race. +set -eu + +KEYCLOAK_URL="http://keycloak:8080" +ADMIN_USERNAME="${KEYCLOAK_ADMIN_USERNAME:-admin}" +ADMIN_PASSWORD="${KEYCLOAK_ADMIN_PASSWORD:-admin}" + +echo "Waiting for Keycloak to be ready..." +until curl -sf "${KEYCLOAK_URL}/realms/master/.well-known/openid-configuration" -o /dev/null; do + sleep 2 +done + +echo "Keycloak is up. Importing realms from /realms..." + +TOKEN=$(curl -sf -X POST "${KEYCLOAK_URL}/realms/master/protocol/openid-connect/token" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=password" \ + -d "client_id=admin-cli" \ + -d "username=${ADMIN_USERNAME}" \ + -d "password=${ADMIN_PASSWORD}" \ + | sed -n 's/.*"access_token":"\([^"]*\)".*/\1/p') + +if [ -z "$TOKEN" ]; then + echo "Failed to obtain an admin token" >&2 + exit 1 +fi + +for realm_file in /realms/*.json; do + [ -e "$realm_file" ] || continue + echo "Importing ${realm_file}..." + http_code=$(curl -s -o /tmp/import-response.txt -w "%{http_code}" \ + -X POST "${KEYCLOAK_URL}/admin/realms" \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + --data-binary "@${realm_file}") + case "$http_code" in + 201) echo " created" ;; + 409) echo " already exists, skipping" ;; + *) + echo " unexpected HTTP ${http_code}:" + cat /tmp/import-response.txt + exit 1 + ;; + esac +done + +echo "Realm import complete." diff --git a/docker/keycloak/realms/account-api-realm.json b/docker/keycloak/realms/account-api-realm.json new file mode 100644 index 0000000..faa63ad --- /dev/null +++ b/docker/keycloak/realms/account-api-realm.json @@ -0,0 +1,24 @@ +{ + "realm": "sisques-account", + "enabled": true, + "registrationAllowed": false, + "loginWithEmailAllowed": true, + "duplicateEmailsAllowed": false, + "sslRequired": "none", + "clients": [ + { + "clientId": "account-api", + "name": "Sisques Account API (internal adapter)", + "enabled": true, + "protocol": "openid-connect", + "publicClient": false, + "secret": "local-dev-secret-change-me", + "standardFlowEnabled": false, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": true, + "authorizationServicesEnabled": false, + "fullScopeAllowed": true + } + ] +}