diff --git a/.env.template b/.env.template index 919d2e2..f8a298f 100644 --- a/.env.template +++ b/.env.template @@ -9,7 +9,7 @@ DATABASE_POOL_SIZE=10 DATABASE_MAX_OVERFLOW=5 DATABASE_ECHO=false -REDIS_URL= +REDIS_URL=redis://127.0.0.1:56479/0 ENTRA_TENANT_ID= ENTRA_API_CLIENT_ID= diff --git a/.github/workflows/backend-ci.yml b/.github/workflows/backend-ci.yml new file mode 100644 index 0000000..fbd7a7b --- /dev/null +++ b/.github/workflows/backend-ci.yml @@ -0,0 +1,96 @@ +name: backend-ci + +on: + push: + branches: [main] + pull_request: + +concurrency: + group: backend-ci-${{ github.ref }} + cancel-in-progress: true + +env: + UV_FROZEN: "1" + +jobs: + static: + name: lint & typecheck + runs-on: ubuntu-latest + defaults: + run: + working-directory: backend + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: backend/uv.lock + + - name: Install dependencies + run: uv sync --all-groups + + - name: Ruff lint + run: uv run ruff check . + + - name: Ruff format + run: uv run ruff format --check . + + - name: Mypy + run: uv run mypy src + + test: + name: test + runs-on: ubuntu-latest + defaults: + run: + working-directory: backend + + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: generate_admin + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d generate_admin" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 3s + --health-retries 10 + + env: + # Ports are the container defaults here, not the offset ones docker-compose + # publishes locally. Everything else falls back to its config default. + DATABASE_URL: postgresql+asyncpg://postgres:postgres@127.0.0.1:5432/generate_admin + REDIS_URL: redis://127.0.0.1:6379/0 + APP_ENVIRONMENT: local + + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: backend/uv.lock + + - name: Install dependencies + run: uv sync --all-groups + + - name: Run migrations + run: uv run alembic upgrade head + + - name: Pytest + run: uv run pytest diff --git a/README.md b/README.md index 98e2819..07cd8db 100644 --- a/README.md +++ b/README.md @@ -1 +1,59 @@ -# admin \ No newline at end of file +# Generate Admin + +## Requirements + +- [uv](https://docs.astral.sh/uv/) (Python 3.12+) +- Docker, for Postgres / Redis / LocalStack +- [just](https://github.com/casey/just) + +## Quickstart + +```bash +cp .env.template .env # defaults already match docker-compose +just install +just up # postgres + redis + localstack +just migrate +just dev # http://localhost:8000 +``` + +Check it with `curl localhost:8000/health`. API docs are at `/docs`. + +## Services + +Docker compose uses offset host ports so it does not collide with anything already running. + +| Service | Host port | Notes | +| ---------- | --------- | -------------------------------------------- | +| Postgres | `55532` | db `generate_admin`, user/password `postgres` | +| Redis | `56479` | cache only, no persistence | +| LocalStack | `4576` | S3 only, bucket `generate-admin-local` | + +## Commands + +| Recipe | What it does | +| ------------------ | -------------------------------- | +| `just up` / `down` | start / stop docker services | +| `just reset` | drop the Postgres volume, restart | +| `just dev` | run the API with reload | +| `just migrate` | apply migrations | +| `just rollback` | undo the last migration | +| `just revision m` | create a migration | +| `just test` | pytest | +| `just lint` | ruff check + format check | +| `just fmt` | ruff autofix + format | +| `just typecheck` | mypy over `src` | +| `just check` | lint + typecheck + test | + +## Tests + +`just test` needs Postgres and Redis running, since the suite boots the real app lifespan and +uses a real Redis. + +## CI + +`.github/workflows/backend-ci.yml` runs on pushes to `main` and on every PR: + +- lint and typecheck: ruff and mypy +- test: Postgres and Redis service containers, migrations, then pytest + +It mirrors `just check`. diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 5815d73..c8dd89e 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -51,6 +51,11 @@ plugins = ["pydantic.mypy"] strict = true warn_return_any = false +# These ship without type stubs; everything else stays strict. +[[tool.mypy.overrides]] +module = ["asyncpg.*", "boto3.*", "botocore.*"] +ignore_missing_imports = true + [tool.pytest.ini_options] asyncio_mode = "auto" testpaths = ["tests"] diff --git a/backend/src/generate_admin/core/cache.py b/backend/src/generate_admin/core/cache.py index a3932ad..82cfc23 100644 --- a/backend/src/generate_admin/core/cache.py +++ b/backend/src/generate_admin/core/cache.py @@ -1,8 +1,5 @@ import asyncio -import time -from collections import defaultdict from collections.abc import Awaitable, Callable -from dataclasses import dataclass from enum import StrEnum from typing import Any, Protocol @@ -55,84 +52,6 @@ async def lock_for(self, key: str) -> asyncio.Lock: return self._locks.setdefault(key, asyncio.Lock()) -@dataclass(slots=True) -class Entry: - value: Any - expires_at: float - - -class InProcessCache: - def __init__( - self, *, default_ttl: float = DEFAULT_TTL_SECONDS, max_entries: int = 4096 - ) -> None: - self._entries: dict[str, Entry] = {} - self._versions: dict[CacheNamespace, int] = defaultdict(int) - self._flight = SingleFlight() - self._default_ttl = default_ttl - self._max_entries = max_entries - - def _qualified(self, namespace: CacheNamespace, key: str) -> str: - return f"{namespace.value}:v{self._versions[namespace]}:{key}" - - async def bump(self, namespace: CacheNamespace) -> None: - self._versions[namespace] += 1 - - async def version(self, namespace: CacheNamespace) -> int: - return self._versions[namespace] - - async def close(self) -> None: - self._entries.clear() - - def _read(self, qualified: str) -> Any | None: - entry = self._entries.get(qualified) - if entry is None: - return None - if time.monotonic() >= entry.expires_at: - self._entries.pop(qualified, None) - return None - return entry.value - - def _evict(self) -> None: - now = time.monotonic() - for key in [key for key, entry in self._entries.items() if entry.expires_at <= now]: - self._entries.pop(key, None) - - overflow = len(self._entries) - self._max_entries + 1 - if overflow > 0: - expiring_first = sorted(self._entries.items(), key=lambda item: item[1].expires_at) - for key, _ in expiring_first[:overflow]: - self._entries.pop(key, None) - - async def fetch( - self, - namespace: CacheNamespace, - key: str, - loader: Loader, - *, - adapter: TypeAdapter[Any], - ttl: float | None = None, - ) -> Any: - qualified = self._qualified(namespace, key) - - cached = self._read(qualified) - if cached is not None: - return cached - - lock = await self._flight.lock_for(qualified) - async with lock: - cached = self._read(qualified) - if cached is not None: - return cached - - value = await loader() - if len(self._entries) >= self._max_entries: - self._evict() - self._entries[qualified] = Entry( - value=value, expires_at=time.monotonic() + (ttl or self._default_ttl) - ) - return value - - class RedisCache: def __init__(self, client: Redis, *, default_ttl: float = DEFAULT_TTL_SECONDS) -> None: self._client = client @@ -185,11 +104,7 @@ async def fetch( async def build_cache(redis_url: str, *, default_ttl: float = DEFAULT_TTL_SECONDS) -> Cache: if not redis_url: - logger.warning( - "cache_in_process", - detail="REDIS_URL is unset; cache is per-process and unsafe with multiple workers", - ) - return InProcessCache(default_ttl=default_ttl) + raise RuntimeError("REDIS_URL is required") client: Redis = Redis.from_url(redis_url, decode_responses=False) try: diff --git a/backend/src/generate_admin/core/storage.py b/backend/src/generate_admin/core/storage.py index 7f87868..8142f51 100644 --- a/backend/src/generate_admin/core/storage.py +++ b/backend/src/generate_admin/core/storage.py @@ -20,6 +20,7 @@ class MediaVisibility(StrEnum): PUBLIC = "public" PRIVATE = "private" + PRESIGN_CACHE_RATIO = 0.8 PUBLIC_PREFIX = "public" PRIVATE_PREFIX = "private" diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 0000000..272ad43 --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,19 @@ +from collections.abc import Iterator + +import pytest +from fastapi.testclient import TestClient + +from generate_admin.core.config import Settings, get_settings +from generate_admin.main import create_app + + +@pytest.fixture(scope="session") +def settings() -> Settings: + return get_settings() + + +@pytest.fixture(scope="session") +def client() -> Iterator[TestClient]: + """Runs the real lifespan, so this needs Postgres and Redis to be up.""" + with TestClient(create_app()) as test_client: + yield test_client diff --git a/backend/tests/test_cache.py b/backend/tests/test_cache.py new file mode 100644 index 0000000..75f5aaf --- /dev/null +++ b/backend/tests/test_cache.py @@ -0,0 +1,79 @@ +import asyncio +import uuid + +import pytest +from pydantic import TypeAdapter + +from generate_admin.core.cache import Cache, CacheNamespace, build_cache +from generate_admin.core.config import Settings + +STRING_ADAPTER = TypeAdapter(str) + + +@pytest.fixture +async def cache(settings: Settings) -> Cache: + return await build_cache(settings.redis_url) + + +def unique_key() -> str: + return f"test:{uuid.uuid4().hex}" + + +async def test_build_cache_requires_a_url() -> None: + with pytest.raises(RuntimeError, match="REDIS_URL is required"): + await build_cache("") + + +async def test_second_fetch_is_served_from_cache(cache: Cache) -> None: + key = unique_key() + calls = 0 + + async def loader() -> str: + nonlocal calls + calls += 1 + return "value" + + first = await cache.fetch(CacheNamespace.CONTENT, key, loader, adapter=STRING_ADAPTER) + second = await cache.fetch(CacheNamespace.CONTENT, key, loader, adapter=STRING_ADAPTER) + + assert first == second == "value" + assert calls == 1 + + +async def test_concurrent_misses_run_the_loader_once(cache: Cache) -> None: + key = unique_key() + calls = 0 + + async def loader() -> str: + nonlocal calls + calls += 1 + await asyncio.sleep(0.05) + return "value" + + results = await asyncio.gather( + *( + cache.fetch(CacheNamespace.CONTENT, key, loader, adapter=STRING_ADAPTER) + for _ in range(10) + ) + ) + + assert results == ["value"] * 10 + assert calls == 1 + + +async def test_bump_invalidates_the_whole_namespace(cache: Cache) -> None: + key = unique_key() + calls = 0 + + async def loader() -> str: + nonlocal calls + calls += 1 + return f"value-{calls}" + + before = await cache.version(CacheNamespace.ROLES) + assert await cache.fetch(CacheNamespace.ROLES, key, loader, adapter=STRING_ADAPTER) == "value-1" + + await cache.bump(CacheNamespace.ROLES) + + assert await cache.version(CacheNamespace.ROLES) == before + 1 + assert await cache.fetch(CacheNamespace.ROLES, key, loader, adapter=STRING_ADAPTER) == "value-2" diff --git a/backend/tests/test_health.py b/backend/tests/test_health.py new file mode 100644 index 0000000..025d840 --- /dev/null +++ b/backend/tests/test_health.py @@ -0,0 +1,15 @@ +from fastapi.testclient import TestClient + + +def test_health_reports_database_up(client: TestClient) -> None: + response = client.get("/health") + + assert response.status_code == 200 + assert response.json() == {"status": "ok", "database": "up"} + + +def test_app_wires_cache_and_storage(client: TestClient) -> None: + state = client.app.state + + assert type(state.cache).__name__ == "RedisCache" + assert state.storage is not None