diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml index 1e3dcdeb1..9a8a9e7f8 100644 --- a/.docker/docker-compose.yml +++ b/.docker/docker-compose.yml @@ -212,10 +212,10 @@ services: REPOSITORY_NAME: "${REPOSITORY_NAME:-Database Repository}" ROR_ENDPOINT: "${ROR_ENDPOINT:-https://api.ror.org}" SEARCH_SERVICE_ENDPOINT: "${SEARCH_SERVICE_ENDPOINT:-http://search-service:8080}" - S3_ACCESS_KEY_ID: "${S3_USER_ACCESS_KEY_ID:-seaweedfsuser}" + S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsuser}" S3_BUCKET: "${S3_BUCKET:-dbrepo}" S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}" - S3_SECRET_ACCESS_KEY: "${S3_USER_SECRET_ACCESS_KEY:-seaweedfsuser}" + S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsuser}" SPARQL_CONNECTION_TIMEOUT: "${SPARQL_CONNECTION_TIMEOUT:-10000}" SYSTEM_USERNAME: "${SYSTEM_USERNAME:-admin}" SYSTEM_PASSWORD: "${SYSTEM_PASSWORD:-admin}" @@ -574,10 +574,10 @@ services: QUEUE_NAME: ${QUEUE_NAME:-dbrepo} REQUEUE_REJECTED: ${REQUEUE_REJECTED:-false} ROUTING_KEY: "${ROUTING_KEY:-dbrepo.#}" - S3_ACCESS_KEY_ID: "${S3_USER_ACCESS_KEY_ID:-seaweedfsuser}" + S3_ACCESS_KEY: "${S3_ACCESS_KEY_ID:-seaweedfsuser}" S3_BUCKET: "${S3_BUCKET:-dbrepo}" S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}" - S3_SECRET_ACCESS_KEY: "${S3_USER_SECRET_ACCESS_KEY:-seaweedfsuser}" + S3_SECRET_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsuser}" SPARK_USER: "${COMPUTE_SERVICE_USERNAME:-spark}" SYSTEM_USERNAME: "${SYSTEM_USERNAME:-admin}" SYSTEM_PASSWORD: "${SYSTEM_PASSWORD:-admin}" diff --git a/.scripts/gen-secrets.sh b/.scripts/gen-secrets.sh index bb004aab1..e0d188738 100644 --- a/.scripts/gen-secrets.sh +++ b/.scripts/gen-secrets.sh @@ -194,6 +194,11 @@ SECRET_PATH="./dbrepo-metadata-db" if [[ $INSTALL_SCRIPT -eq 1 ]]; then SECRET_PATH="./config" fi +cat < "$SECRET_PATH/metrics.cnf" +[client] +user=readonly +password=$READONLY_PASSWORD +EOF cat < $SECRET_PATH/2_setup-data.sql BEGIN; INSERT INTO \`mdb_containers\` (id, name, internal_name, image_id, host, port, ui_host, ui_port, privileged_username, diff --git a/.scripts/test-compose-credentials.py b/.scripts/test-compose-credentials.py new file mode 100644 index 000000000..1616d6ae4 --- /dev/null +++ b/.scripts/test-compose-credentials.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Run with python3 .scripts/test-compose-credentials.py; requires Docker Compose, not a daemon.""" + +import configparser +import json +import os +from pathlib import Path +import subprocess +import tempfile + + +ROOT = Path(__file__).resolve().parents[1] +# Do not inherit deployment secrets or Compose overrides from the calling shell. +ENV = {key: os.environ[key] for key in ("PATH", "HOME") if key in os.environ} + + +def check(): + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + env_file = work / ".env" + for custom in (False, True): + credentials = { + "S3_ACCESS_KEY_ID": "test-user-key", + "S3_SECRET_ACCESS_KEY": "test-user-secret", + "S3_ADMIN_ACCESS_KEY_ID": "test-admin-key", + "S3_ADMIN_SECRET_ACCESS_KEY": "test-admin-secret", + } if custom else {} + env_file.write_text("".join(f"{key}={value}\n" for key, value in credentials.items())) + for compose in ("docker-compose.yml", ".docker/docker-compose.yml"): + result = subprocess.run( + ["docker", "compose", "--env-file", str(env_file), "-f", + str(ROOT / compose), "config", "--format", "json"], + env=ENV, capture_output=True, text=True, check=True, + ) + services = json.loads(result.stdout)["services"] + for service, names, source_names, fallback in ( + ("dbrepo-metadata-service", ("S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"), + ("S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"), "seaweedfsuser"), + ("dbrepo-data-service", ("S3_ACCESS_KEY", "S3_SECRET_KEY"), + ("S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"), "seaweedfsuser"), + ("dbrepo-storage-service-init", ("S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"), + ("S3_ADMIN_ACCESS_KEY_ID", "S3_ADMIN_SECRET_ACCESS_KEY"), "seaweedfsadmin"), + ): + for name, source_name in zip(names, source_names): + assert services[service]["environment"].get(name) == credentials.get(source_name, fallback), (compose, service, name) + + # Execute only database config generation: no secret rotation, TLS creation or sudo. + script = (ROOT / ".scripts/gen-secrets.sh").read_text() + database_setup = script[script.index('SECRET_PATH="./dbrepo-metadata-db"'):] + for install, target in (("0", "dbrepo-metadata-db"), ("1", "config")): + (work / target).mkdir() + subprocess.run( + ["bash", "-eu", "-c", database_setup], cwd=work, check=True, + env={**ENV, "INSTALL_SCRIPT": install, "READONLY_PASSWORD": "test-readonly-password", + "DATA_DB_PASSWORD": "test-root-password"}, + ) + metrics = configparser.ConfigParser() + metrics.read(work / target / "metrics.cnf") + assert metrics["client"]["user"] == "readonly" + assert metrics["client"]["password"] == "test-readonly-password" + print("PASS: both Compose files, default/custom S3 credentials, admin credentials, and both metrics output paths") + + +if __name__ == "__main__": + check() diff --git a/docker-compose.yml b/docker-compose.yml index 82a2c992e..1d1d3c1f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -179,10 +179,10 @@ services: REPOSITORY_NAME: "${REPOSITORY_NAME:-Database Repository}" ROR_ENDPOINT: "${ROR_ENDPOINT:-https://api.ror.org}" SEARCH_SERVICE_ENDPOINT: "${SEARCH_SERVICE_ENDPOINT:-http://search-service:8080}" - S3_ACCESS_KEY_ID: "${S3_USER_ACCESS_KEY_ID:-seaweedfsuser}" + S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsuser}" S3_BUCKET: "${S3_BUCKET:-dbrepo}" S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}" - S3_SECRET_ACCESS_KEY: "${S3_USER_SECRET_ACCESS_KEY:-seaweedfsuser}" + S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsuser}" SPARQL_CONNECTION_TIMEOUT: "${SPARQL_CONNECTION_TIMEOUT:-10000}" SYSTEM_USERNAME: "${SYSTEM_USERNAME:-admin}" SYSTEM_PASSWORD: "${SYSTEM_PASSWORD:-admin}" @@ -557,10 +557,10 @@ services: QUEUE_NAME: ${QUEUE_NAME:-dbrepo} REQUEUE_REJECTED: ${REQUEUE_REJECTED:-false} ROUTING_KEY: "${ROUTING_KEY:-dbrepo.#}" - S3_ACCESS_KEY_ID: "${S3_USER_ACCESS_KEY_ID:-seaweedfsuser}" + S3_ACCESS_KEY: "${S3_ACCESS_KEY_ID:-seaweedfsuser}" S3_BUCKET: "${S3_BUCKET:-dbrepo}" S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}" - S3_SECRET_ACCESS_KEY: "${S3_USER_SECRET_ACCESS_KEY:-seaweedfsuser}" + S3_SECRET_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsuser}" SPARK_USER: "${COMPUTE_SERVICE_USERNAME:-spark}" SYSTEM_USERNAME: "${SYSTEM_USERNAME:-admin}" SYSTEM_PASSWORD: "${SYSTEM_PASSWORD:-admin}"