diff --git a/.env.example b/.env.example index 68c32645..fa5bd726 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,17 @@ NO_LIMITS=true DEBUG_ENDPOINTS=true ACTIVITY_SHOW_AUTHORIZED_PHOTOS=false WORKER_URL=http://localhost:8056 + +# Panoramax federation container (optional — compose profile "panoramax"). +# Password for the dedicated read-mostly DB role; on fresh clusters the role is +# created by initdb, on existing ones run backend/scripts/provision_panoramax_role.sh. +# PANORAMAX_DB_PASSWORD=change-me +# Public canonical URL registered in the meta-catalog. Its last characters must +# not be '/', 'a', 'p' or 'i' (the catalog's canonical_url() rstrip("/api")s the +# whole string) — any *.hillview.cz host is fine since 'z' isn't in that set. +# PANORAMAX_BASE_URL=https://cc.geovisio.hillview.cz +# Where "/" redirects humans arriving from the catalog's rel=via link +# PANORAMAX_VIEWER_URL=https://hillview.cz PICS_URL=http://localhost:9999/ # Storage pools (optional). FILE_POOLS is a JSON array describing every location diff --git a/CLAUDE.md b/CLAUDE.md index 8375c9d4..b0339dcf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -112,6 +112,7 @@ Each subdirectory has its own `CLAUDE.md` with detailed instructions: - **[Terrain Data Licensing](docs/terrain-data-licensing.md)**: DEM/OSM licence obligations for terrain renders (required notices, pre-launch checklist) - **[Native Android Auth](docs/native-auth.md)**: Credential Manager + Google ID-token login — concepts, security reasoning, and where everything lives - **[Zoom view print view](docs/zoomview-print.md)**: ⋮ → Print view + Ctrl+P — share-link QR in the middle, why the viewer freezes instead of re-rendering at print time, the replaced-element canvas gotcha +- **[Panoramax Federation](docs/panoramax-federation.md)**: The `backend/panoramax/` read API + sequencer serving CC photos to the Panoramax federation (harvester contract, deployment, registration) ## Common Issues & Solutions diff --git a/backend/api/app/alembic/versions/030_add_panoramax_schema.py b/backend/api/app/alembic/versions/030_add_panoramax_schema.py new file mode 100644 index 00000000..f8183975 --- /dev/null +++ b/backend/api/app/alembic/versions/030_add_panoramax_schema.py @@ -0,0 +1,185 @@ +"""Add the `panoramax` PG schema: synthesized sequences for the Panoramax federation. + +Hillview joins the Panoramax federation by serving a GeoVisio-compatible read +API (backend/panoramax/) whose "collections" are sequences synthesized from +users' photos by per-owner time-gap session splitting. This migration is purely +additive to the existing schema: the only touch on existing tables is an AFTER +UPDATE trigger on photos (+ FKs from the new tables). + +Design constraints (from the meta-catalog harvester, see docs/panoramax-federation.md): +- Sequence ids must be real UUIDs — the meta-catalog casts `content->>'id'` to + UUID primary-key columns. +- Tombstones are never hard-deleted: a sequence that loses all members flips to + status='deleted' and must keep being served (the harvester's incremental sync + lists `status IN ('deleted','ready') AND updated > ` — the updated_at bump + is the only channel through which deletions propagate to the catalog). +- owner_id is ON DELETE SET NULL so tombstones survive account deletion. +- UNIQUE(photo_id) on membership: a photo belongs to at most one sequence, which + stays correct across future scopes because scopes partition by license. +- The (scope, updated_at) index serves the harvester's incremental crawl filter. +- Membership triggers fire on cascaded deletes too (PG fires row triggers on the + referencing table when a photos hard-delete cascades), so photo hard-deletes + bump/tombstone sequences without any app-side code. + +NOT here by design: backfill (the sequencer's first run does it) and role +creation (panoramax_ro is provisioning/initdb territory, not alembic — see +docker/postgres/). Grants ARE applied here, guarded on the role's existence, +because on a fresh cluster the schema doesn't exist yet at initdb time. + +Revision ID: 030_add_panoramax_schema +Revises: 029_share_links +Create Date: 2026-08-05 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = '030_add_panoramax_schema' +down_revision: Union[str, None] = '029_share_links' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute("CREATE SCHEMA IF NOT EXISTS panoramax") + + op.execute(""" + CREATE TABLE panoramax.sequences ( + id UUID PRIMARY KEY, + scope VARCHAR NOT NULL, + status VARCHAR NOT NULL DEFAULT 'ready' + CONSTRAINT sequences_status_check CHECK (status IN ('ready', 'deleted')), + owner_id VARCHAR REFERENCES users(id) ON DELETE SET NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """) + # Incremental-crawl index: the harvester filters by scope-wide status + + # `updated > `; scope leads so a second instance scope stays cheap. + op.execute(""" + CREATE INDEX ix_panoramax_sequences_scope_updated + ON panoramax.sequences (scope, updated_at) + """) + op.execute(""" + CREATE INDEX ix_panoramax_sequences_owner + ON panoramax.sequences (owner_id) + """) + + # PK is photo_id (globally unique membership). The (sequence_id, rank) + # uniqueness is DEFERRABLE so the sequencer can renumber ranks within a + # transaction without transient collisions. + op.execute(""" + CREATE TABLE panoramax.sequence_photos ( + photo_id VARCHAR PRIMARY KEY REFERENCES photos(id) ON DELETE CASCADE, + sequence_id UUID NOT NULL REFERENCES panoramax.sequences(id) ON DELETE CASCADE, + rank INTEGER NOT NULL, + CONSTRAINT sequence_photos_rank_unique UNIQUE (sequence_id, rank) + DEFERRABLE INITIALLY DEFERRED + ) + """) + op.execute(""" + CREATE INDEX ix_panoramax_sequence_photos_seq_rank + ON panoramax.sequence_photos (sequence_id, rank) + """) + + # Any change to a member photo that alters what the federation sees + # (visibility, license, position, heading, capture time, derivatives, + # title/description, processing state, soft-delete) bumps the owning + # sequence's updated_at so the harvester re-crawls that collection. + # geometry is compared as text (exact EWKB hex; PostGIS `=` is bbox + # equality) and sizes as text (json has no equality operator). + op.execute(""" + CREATE FUNCTION panoramax.bump_sequence_on_photo_change() RETURNS trigger AS $$ + BEGIN + IF (OLD.deleted IS DISTINCT FROM NEW.deleted + OR OLD.is_public IS DISTINCT FROM NEW.is_public + OR OLD.legal_rights IS DISTINCT FROM NEW.legal_rights + OR OLD.geometry::text IS DISTINCT FROM NEW.geometry::text + OR OLD.compass_angle IS DISTINCT FROM NEW.compass_angle + OR OLD.captured_at IS DISTINCT FROM NEW.captured_at + OR OLD.effective_at IS DISTINCT FROM NEW.effective_at + OR OLD.sizes::text IS DISTINCT FROM NEW.sizes::text + OR OLD.title IS DISTINCT FROM NEW.title + OR OLD.description IS DISTINCT FROM NEW.description + OR OLD.processing_status IS DISTINCT FROM NEW.processing_status) THEN + UPDATE panoramax.sequences s + SET updated_at = now() + FROM panoramax.sequence_photos sp + WHERE sp.photo_id = NEW.id AND s.id = sp.sequence_id; + END IF; + RETURN NULL; + END; + $$ LANGUAGE plpgsql; + """) + op.execute(""" + CREATE TRIGGER panoramax_photo_change_trg + AFTER UPDATE ON photos + FOR EACH ROW EXECUTE FUNCTION panoramax.bump_sequence_on_photo_change(); + """) + + # Membership changes bump the sequence, and a sequence emptied by deletes is + # tombstoned (status='deleted'), never removed. Covers the sequencer's own + # writes AND cascaded deletes from photos/users hard-deletes. + op.execute(""" + CREATE FUNCTION panoramax.bump_sequence_on_membership() RETURNS trigger AS $$ + BEGIN + IF TG_OP IN ('INSERT', 'UPDATE') THEN + -- a sequence gaining a member is live by definition: revive + -- tombstones the sequencer repopulates, and bump updated_at + UPDATE panoramax.sequences + SET updated_at = now(), status = 'ready' + WHERE id = NEW.sequence_id; + END IF; + IF TG_OP IN ('UPDATE', 'DELETE') + AND (TG_OP = 'DELETE' OR OLD.sequence_id IS DISTINCT FROM NEW.sequence_id) THEN + UPDATE panoramax.sequences + SET updated_at = now() + WHERE id = OLD.sequence_id; + UPDATE panoramax.sequences s + SET status = 'deleted', updated_at = now() + WHERE s.id = OLD.sequence_id + AND s.status <> 'deleted' + AND NOT EXISTS ( + SELECT 1 FROM panoramax.sequence_photos sp + WHERE sp.sequence_id = OLD.sequence_id + ); + END IF; + RETURN NULL; + END; + $$ LANGUAGE plpgsql; + """) + op.execute(""" + CREATE TRIGGER panoramax_membership_trg + AFTER INSERT OR UPDATE OR DELETE ON panoramax.sequence_photos + FOR EACH ROW EXECUTE FUNCTION panoramax.bump_sequence_on_membership(); + """) + + # Grants for the dedicated read-mostly role, applied only if provisioning + # already created it (fresh clusters: docker/postgres/initdb.d creates the + # role before the api container ever runs alembic; existing deployments: + # scripts/provision_panoramax_role.sh, which re-applies these grants itself). + op.execute(""" + DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'panoramax_ro') THEN + GRANT USAGE ON SCHEMA public TO panoramax_ro; + GRANT SELECT ON photos, users, photo_ratings, flagged_photos + TO panoramax_ro; + GRANT USAGE ON SCHEMA panoramax TO panoramax_ro; + GRANT SELECT, INSERT, UPDATE, DELETE + ON panoramax.sequences, panoramax.sequence_photos TO panoramax_ro; + END IF; + END $$; + """) + + +def downgrade() -> None: + op.execute("DROP TRIGGER IF EXISTS panoramax_photo_change_trg ON photos") + op.execute("DROP FUNCTION IF EXISTS panoramax.bump_sequence_on_photo_change()") + op.execute("DROP TRIGGER IF EXISTS panoramax_membership_trg ON panoramax.sequence_photos") + op.execute("DROP FUNCTION IF EXISTS panoramax.bump_sequence_on_membership()") + op.execute("DROP TABLE IF EXISTS panoramax.sequence_photos") + op.execute("DROP TABLE IF EXISTS panoramax.sequences") + op.execute("DROP SCHEMA IF EXISTS panoramax") diff --git a/backend/panoramax/Dockerfile b/backend/panoramax/Dockerfile new file mode 100644 index 00000000..a92e3d28 --- /dev/null +++ b/backend/panoramax/Dockerfile @@ -0,0 +1,40 @@ +# Panoramax federation read API. Build context is backend/ (like api and +# worker), same uv-export pattern as worker/Dockerfile. +FROM python:3.12-slim + +RUN apt-get update && apt-get install -y \ + gcc \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=ghcr.io/astral-sh/uv:0.11.5 /uv /bin/uv + +WORKDIR /app + +# Workspace root + member pyproject files needed to resolve this package +COPY pyproject.toml uv.lock /app/ +COPY common/pyproject.toml /app/common/ +COPY panoramax/pyproject.toml /app/panoramax/ + +RUN uv export --frozen --no-hashes --no-emit-project --package hillview-panoramax | \ + grep -v "sys_platform == 'darwin'" | \ + sed "s/ ; sys_platform != 'darwin'//" > /tmp/requirements.txt && \ + uv pip install --system --no-deps -r /tmp/requirements.txt && \ + rm /tmp/requirements.txt + +RUN groupadd --gid 1001 panoramax && \ + useradd --create-home --shell /bin/bash --uid 1001 --gid 1001 panoramax + +COPY panoramax/app /app/app + +RUN python -m compileall -b /app/app && chown -R panoramax:panoramax /app + +USER panoramax +WORKDIR /app/app + +ENV PYTHONPATH="/app/app:/app" + +# 0.0.0.0, not "::": under uvloop (no --reload) a "::" bind is IPv6-only, +# which dead-ends the bridge-mode port mapping (docker-proxy connects over +# IPv4). External IPv6 is Caddy's job. +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8058"] diff --git a/backend/panoramax/app/__init__.py b/backend/panoramax/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/panoramax/app/cql.py b/backend/panoramax/app/cql.py new file mode 100644 index 00000000..87e462f9 --- /dev/null +++ b/backend/panoramax/app/cql.py @@ -0,0 +1,137 @@ +"""CQL2-text `filter` parameter of /api/collections. + +Parsing is delegated to pygeofilter — the same library the reference GeoVisio +server uses for its own `filter` parameters (geovisio/utils/cql2.py) — so the +grammar is the real CQL2 grammar, not a home-grown approximation. What we do +here is walk the resulting AST and accept only the subset we can execute: + + status IN ('deleted','ready') AND updated > '2026-01-01T00:00:00Z' + +which is the one shape the meta-catalog harvester sends (harvest.py +get_collections), plus small variations (`status = '...'`, `>=`, clauses in +any order, `TIMESTAMP('...')` literals, parentheses). Anything else — other +attributes, OR/NOT, other operators, non-literal operands — is rejected with +a FilterParseError so a client speaking more CQL than we execute fails loudly +instead of silently getting an unfiltered listing. +""" +from dataclasses import dataclass +from datetime import date, datetime, timezone +from typing import Iterator + +from pygeofilter import ast +from pygeofilter.parsers.cql2_text import parse as _parse_cql2_text + +VALID_STATUSES = {'ready', 'deleted'} + + +class FilterParseError(ValueError): + pass + + +@dataclass +class CollectionsFilter: + # None = clause absent (defaults applied by the caller), else the allowed set + statuses: set[str] | None = None + updated_after: datetime | None = None + updated_inclusive: bool = False + + +def parse_collections_filter(raw: str | None) -> CollectionsFilter: + result = CollectionsFilter() + if raw is None or raw.strip() == '': + return result + try: + tree = _parse_cql2_text(raw) + except Exception as e: # lark UnexpectedToken/UnexpectedCharacters, literal conversion errors, ... + # first line only: lark appends its full expected-token list, which is + # grammar internals, not something a 400 body should carry + reason = (str(e).strip().splitlines() or ['unparseable'])[0] + raise FilterParseError(f"malformed CQL2 filter: {reason}") from e + for clause in _conjuncts(tree): + _apply_clause(clause, result) + return result + + +def _conjuncts(node: ast.Node) -> Iterator[ast.Node]: + """Flatten a (left-nested) AND tree into its top-level clauses.""" + if isinstance(node, ast.And): + yield from _conjuncts(node.lhs) + yield from _conjuncts(node.rhs) + else: + yield node + + +def _apply_clause(node: ast.Node, result: CollectionsFilter) -> None: + if isinstance(node, ast.In): + if _attribute(node.lhs) != 'status': + raise FilterParseError(f"unsupported attribute in IN clause: {_describe(node.lhs)}") + if node.not_: + raise FilterParseError("NOT IN is not supported") + _set_statuses(result, {_status_literal(v) for v in node.sub_nodes}) + elif isinstance(node, ast.Equal): + if _attribute(node.lhs) != 'status': + raise FilterParseError(f"unsupported attribute in = clause: {_describe(node.lhs)}") + _set_statuses(result, {_status_literal(node.rhs)}) + elif isinstance(node, (ast.GreaterThan, ast.GreaterEqual)): + if _attribute(node.lhs) != 'updated': + raise FilterParseError(f"unsupported attribute in comparison: {_describe(node.lhs)}") + if result.updated_after is not None: + raise FilterParseError("duplicate updated clause") + result.updated_after = _timestamp_literal(node.rhs) + result.updated_inclusive = isinstance(node, ast.GreaterEqual) + else: + raise FilterParseError(f"unsupported filter clause: {_describe(node)}") + + +def _set_statuses(result: CollectionsFilter, statuses: set[str]) -> None: + if result.statuses is not None: + raise FilterParseError("duplicate status clause") + if not statuses: + raise FilterParseError("empty status list") + result.statuses = statuses + + +def _attribute(node: object) -> str | None: + """Name of an attribute operand (case-folded), or None for anything else.""" + if isinstance(node, ast.Attribute): + return node.name.lower() + return None + + +def _status_literal(value: object) -> str: + # pygeofilter hands quoted strings through as plain str; an unquoted word + # comes back as an Attribute node, a number as int/float + if not isinstance(value, str): + raise FilterParseError(f"status must be a quoted string literal, got {_describe(value)}") + if value not in VALID_STATUSES: + raise FilterParseError(f"unknown status: {value!r}") + return value + + +def _timestamp_literal(value: object) -> datetime: + """A quoted ISO-8601 string (what the harvester sends) or a CQL2 + TIMESTAMP('...') literal, which pygeofilter already turns into a datetime.""" + if isinstance(value, datetime): + dt = value + elif isinstance(value, str): + try: + dt = datetime.fromisoformat(value.replace('Z', '+00:00')) + except ValueError: + raise FilterParseError(f"unparseable timestamp: {value!r}") + else: + # includes date (DATE('...') has no time part, too coarse for a crawl + # cursor) and arithmetic trees such as an unquoted 2026-01-01 + raise FilterParseError(f"timestamp must be a quoted string or TIMESTAMP() literal, got {_describe(value)}") + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + + +def _describe(node: object) -> str: + if isinstance(node, ast.Attribute): + return f"attribute {node.name!r}" + if isinstance(node, ast.Node): + return type(node).__name__ + if isinstance(node, (str, int, float, date, datetime)): + return repr(node) + return type(node).__name__ diff --git a/backend/panoramax/app/db.py b/backend/panoramax/app/db.py new file mode 100644 index 00000000..7f41f5b3 --- /dev/null +++ b/backend/panoramax/app/db.py @@ -0,0 +1,28 @@ +"""Async engine for the panoramax container. + +Deliberately not common.database: this service runs as the dedicated +panoramax_ro role, uses raw SQL only (no ORM models), and must not drag in the +main app's model imports. DATABASE_URL uses the same postgresql+asyncpg:// +scheme as the rest of the backend. +""" +import os + +from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine +from sqlalchemy.pool import NullPool + +_engine: AsyncEngine | None = None + + +def get_engine() -> AsyncEngine: + global _engine + if _engine is None: + url = os.environ['DATABASE_URL'] + kwargs = {} + if os.getenv('DB_NULLPOOL', '').lower() in ('1', 'true', 'yes'): + kwargs['poolclass'] = NullPool + else: + kwargs['pool_size'] = int(os.getenv('DB_POOL_SIZE', '5')) + kwargs['max_overflow'] = int(os.getenv('DB_MAX_OVERFLOW', '5')) + kwargs['pool_pre_ping'] = True + _engine = create_async_engine(url, **kwargs) + return _engine diff --git a/backend/panoramax/app/eligibility.py b/backend/panoramax/app/eligibility.py new file mode 100644 index 00000000..5ad861fb --- /dev/null +++ b/backend/panoramax/app/eligibility.py @@ -0,0 +1,46 @@ +"""The single definition of which photos this instance serves. + +Used by BOTH the sequencer (deciding membership) and the read API (filtering +items at serve time). The serve-time filter matters: when a photo flips out of +scope (soft-delete, is_public off, license change), the photos trigger bumps +the sequence's updated_at so the harvester re-crawls, but until the sequencer +prunes membership the item must already be gone from /items responses. + +Requires `photos p JOIN users u ON u.id = p.owner_id` in the enclosing query +and a :scope_legal_rights bind param. + +Moderation signals also exclude a photo from federation: any thumbs-down +rating, or an unresolved flag (resolved flags don't exclude — an admin looked +and left the photo up; flags resolved by deletion are covered by p.deleted). +Note ratings/flags don't touch the photos row, so they propagate to the +catalog on the sequencer's cadence (membership prune bumps updated_at), while +serve-time filtering hides the item immediately. +""" + +ELIGIBLE_PHOTO_WHERE = """ + p.legal_rights = :scope_legal_rights + AND p.deleted = false + AND p.is_public = true + AND p.processing_status = 'completed' + AND p.geometry IS NOT NULL + AND p.effective_at IS NOT NULL + AND p.sizes IS NOT NULL + AND u.is_active = true + AND u.is_test = false + AND NOT EXISTS ( + SELECT 1 FROM photo_ratings pr + WHERE pr.photo_source = 'hillview' + AND pr.photo_id = p.id + AND pr.rating = 'THUMBS_DOWN' + ) + AND NOT EXISTS ( + SELECT 1 FROM flagged_photos fp + WHERE fp.photo_source = 'hillview' + AND fp.photo_id = p.id + AND fp.resolved = false + ) +""" + +# Deterministic capture order within an owner — same tiebreak as the timeline +# walk (burst shots sharing a 1-second captured_at order by original filename). +PHOTO_ORDER_BY = "p.effective_at, COALESCE(p.original_filename, ''), p.id" diff --git a/backend/panoramax/app/main.py b/backend/panoramax/app/main.py new file mode 100644 index 00000000..2a05f49d --- /dev/null +++ b/backend/panoramax/app/main.py @@ -0,0 +1,416 @@ +"""Panoramax/GeoVisio-compatible read API. + +Serves the three endpoints the meta-catalog harvester actually consumes — +/api/configuration (add-instance aborts without it), /api/collections (CQL2 +status/updated filter + rel=next paging), /api/collections/{id}/items (limit + +rel=next) — plus the STAC landing page and single-resource routes for +completeness. Users/map/RSS/search endpoints are deliberately absent: the +catalog regenerates those itself. + +Read-only over photos/users; sequences come from the panoramax schema +maintained by the in-process sequencer (sequencer.py). +""" +import asyncio +import contextlib +import logging +import uuid as uuid_mod +from typing import Any +from urllib.parse import urlencode + +from fastapi import FastAPI, HTTPException, Query, Request +from fastapi.responses import JSONResponse, RedirectResponse +from sqlalchemy import text + +import sequencer +import settings +from cql import FilterParseError, parse_collections_filter +from db import get_engine +from eligibility import ELIGIBLE_PHOTO_WHERE, PHOTO_ORDER_BY +from stac import STAC_VERSION, collection_json, fmt_dt, item_json + +logger = logging.getLogger('panoramax.api') + +app = FastAPI( + title='Hillview Panoramax API', + description='GeoVisio-compatible read API for the Panoramax federation', + docs_url=None, redoc_url=None, openapi_url=None, +) + + +@app.on_event('startup') +async def _startup() -> None: + if settings.sequencer_enabled(): + app.state.sequencer_task = asyncio.create_task(sequencer.loop(get_engine())) + + +@app.on_event('shutdown') +async def _shutdown() -> None: + task = getattr(app.state, 'sequencer_task', None) + if task: + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + + +def _base_url() -> str: + return settings.base_url() + + +def _as_uuid(value: str, status_code: int = 404) -> uuid_mod.UUID: + """Bind UUIDs as uuid objects (asyncpg array-type inference) and turn + garbage input into a clean HTTP error instead of a driver DataError.""" + try: + return uuid_mod.UUID(value) + except (ValueError, AttributeError, TypeError): + raise HTTPException(status_code=status_code, detail='Not a valid id') + + +@app.get('/') +async def root() -> RedirectResponse: + # Humans arrive here from the catalog's per-item rel=via link, which points + # at the registered instance URL. Send them to the viewer, not to STAC JSON + # (the harvester only ever requests /api/*). + return RedirectResponse(url=settings.viewer_url()) + + +@app.get('/api/health') +async def health() -> dict: + async with get_engine().connect() as conn: + await conn.execute(text('SELECT 1')) + return {'status': 'ok'} + + +@app.get('/api/') +@app.get('/api') +async def landing() -> dict: + base = _base_url() + scope = settings.active_scope() + return { + 'type': 'Catalog', + 'stac_version': STAC_VERSION, + 'id': 'hillview-panoramax', + 'title': settings.instance_name(), + 'description': ( + f"Panoramax-compatible (GeoVisio STAC) view of {settings.instance_name()} " + f"photos published under {scope.license}." + ), + 'conformsTo': [ + 'https://api.stacspec.org/v1.0.0/core', + 'https://api.stacspec.org/v1.0.0/collections', + 'https://api.stacspec.org/v1.0.0/ogcapi-features', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson', + 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/filter', + 'http://www.opengis.net/spec/cql2/1.0/conf/cql2-text', + ], + 'links': [ + {'rel': 'self', 'href': f"{base}/api/", 'type': 'application/json'}, + {'rel': 'root', 'href': f"{base}/api/", 'type': 'application/json'}, + {'rel': 'data', 'href': f"{base}/api/collections", 'type': 'application/json'}, + ], + } + + +@app.get('/api/configuration') +async def configuration() -> dict: + scope = settings.active_scope() + return { + 'name': settings.instance_name(), + # Accuracy matters here — this is not a Panoramax instance. It is + # Hillview serving a Panoramax-compatible read API over the subset of + # its photos whose owners chose the CC license. Note the OSM grant is + # narrower than Panoramax's own CC-BY-SA-4.0 terms (which additionally + # permit derived data under LO 2.0 / CC-BY 4.0 / ODbL 1.0); it is + # spelled out rather than summarised so reviewers can judge it. + 'description': ( + 'Hillview (https://hillview.cz) is a photo mapping application. This ' + 'endpoint is not a Panoramax server: it is a Panoramax-compatible ' + '(GeoVisio STAC) read-only API over the Hillview photos whose owners ' + f'published them under {scope.license}. Photos remain hosted on ' + 'Hillview. Owners additionally grant permission to use their photos as ' + 'reference material for creating, improving or validating OpenStreetMap ' + 'contributions (data so extracted enters OSM under ODbL); that grant ' + 'does not authorise data extraction for other purposes or licenses. ' + 'Full terms: https://hillview.cz/licensing' + ), + 'license': {'id': scope.license, 'url': scope.license_url}, + 'auth': {'enabled': False}, + # The Panoramax mobile app can be pointed at instances that accept + # external contributions; this one is read-only (uploads go through + # hillview.cz). + 'geovisio:external_contributions': False, + } + + +# --- collections ----------------------------------------------------------- + +# Aggregates over *servable* member photos only, so counts/extents never leak +# photos that flipped out of scope between sequencer passes. +_COLLECTION_AGG_SQL = f""" + SELECT + sp.sequence_id, + count(*) AS item_count, + min(p.effective_at) AS min_dt, + max(p.effective_at) AS max_dt, + ST_XMin(ST_Extent(p.geometry)) AS xmin, + ST_YMin(ST_Extent(p.geometry)) AS ymin, + ST_XMax(ST_Extent(p.geometry)) AS xmax, + ST_YMax(ST_Extent(p.geometry)) AS ymax + FROM panoramax.sequence_photos sp + JOIN photos p ON p.id = sp.photo_id + JOIN users u ON u.id = p.owner_id + WHERE sp.sequence_id = ANY(:seq_ids) AND {ELIGIBLE_PHOTO_WHERE} + GROUP BY sp.sequence_id +""" + + +async def _fetch_collections_page( + statuses: set[str], updated_after, updated_inclusive: bool, + limit: int, after_id: str | None, +) -> list[dict[str, Any]]: + scope = settings.active_scope() + base = _base_url() + where = ["s.scope = :scope", "s.status = ANY(:statuses)"] + params: dict[str, Any] = {'scope': scope.id, 'statuses': sorted(statuses), 'limit': limit} + if updated_after is not None: + where.append(f"s.updated_at {'>=' if updated_inclusive else '>'} :updated_after") + params['updated_after'] = updated_after + if after_id: + where.append("s.id > :after_id") + params['after_id'] = _as_uuid(after_id, status_code=400) + + async with get_engine().connect() as conn: + seq_rows = (await conn.execute(text(f""" + SELECT s.id, s.status, s.owner_id, u.username, s.created_at, s.updated_at + FROM panoramax.sequences s + LEFT JOIN users u ON u.id = s.owner_id + WHERE {' AND '.join(where)} + ORDER BY s.id + LIMIT :limit + """), params)).all() + + ready_ids = [r[0] for r in seq_rows if r[1] == 'ready'] + aggs: dict[str, Any] = {} + if ready_ids: + for agg in (await conn.execute( + text(_COLLECTION_AGG_SQL), + {'seq_ids': ready_ids, 'scope_legal_rights': scope.legal_rights}, + )).all(): + aggs[str(agg[0])] = agg + + collections = [] + for r in seq_rows: + seq_id = str(r[0]) + agg = aggs.get(seq_id) + collections.append(collection_json( + seq_id=seq_id, + status=r[1], + owner_id=r[2], + username=r[3], + created_at=r[4], + updated_at=r[5], + item_count=agg[1] if agg else 0, + bbox=[agg[4], agg[5], agg[6], agg[7]] if agg else None, + min_dt=agg[2] if agg else None, + max_dt=agg[3] if agg else None, + license_id=scope.license, + license_url=scope.license_url, + base_url=base, + )) + return collections + + +@app.get('/api/collections') +async def collections( + request: Request, + filter: str | None = Query(default=None), + limit: int = Query(default=settings.COLLECTIONS_PAGE_DEFAULT, ge=1, + le=settings.COLLECTIONS_PAGE_MAX), + page_after: str | None = Query(default=None), +) -> JSONResponse: + try: + parsed = parse_collections_filter(filter) + except FilterParseError as e: + raise HTTPException(status_code=400, detail=str(e)) + # GeoVisio semantics: without an explicit status filter, tombstones are + # hidden; the harvester asks for them explicitly on incremental syncs. + statuses = parsed.statuses or {'ready'} + + page = await _fetch_collections_page( + statuses, parsed.updated_after, parsed.updated_inclusive, + limit, page_after) + + base = _base_url() + links = [ + {'rel': 'self', 'href': str(request.url), 'type': 'application/json'}, + {'rel': 'root', 'href': f"{base}/api/", 'type': 'application/json'}, + ] + if len(page) == limit: + next_params = dict(request.query_params) + next_params['page_after'] = page[-1]['id'] + links.append({ + 'rel': 'next', + 'href': f"{base}/api/collections?{urlencode(next_params)}", + 'type': 'application/json', + }) + return JSONResponse({'collections': page, 'links': links}) + + +async def _load_sequence(seq_id: str): + scope = settings.active_scope() + async with get_engine().connect() as conn: + row = (await conn.execute(text(""" + SELECT s.id, s.status, s.owner_id, u.username, s.created_at, s.updated_at + FROM panoramax.sequences s + LEFT JOIN users u ON u.id = s.owner_id + WHERE s.id = :seq_id AND s.scope = :scope + """), {'seq_id': _as_uuid(seq_id), 'scope': scope.id})).first() + return row + + +@app.get('/api/collections/{seq_id}') +async def collection(seq_id: str) -> dict: + scope = settings.active_scope() + row = await _load_sequence(seq_id) + if row is None: + raise HTTPException(status_code=404, detail='Collection not found') + + agg = None + if row[1] == 'ready': + async with get_engine().connect() as conn: + agg = (await conn.execute( + text(_COLLECTION_AGG_SQL), + {'seq_ids': [row[0]], 'scope_legal_rights': scope.legal_rights}, + )).first() + return collection_json( + seq_id=str(row[0]), + status=row[1], + owner_id=row[2], + username=row[3], + created_at=row[4], + updated_at=row[5], + item_count=agg[1] if agg else 0, + bbox=[agg[4], agg[5], agg[6], agg[7]] if agg else None, + min_dt=agg[2] if agg else None, + max_dt=agg[3] if agg else None, + license_id=scope.license, + license_url=scope.license_url, + base_url=_base_url(), + ) + + +# --- items ----------------------------------------------------------------- + +_ITEMS_SQL = f""" + SELECT + p.id, sp.rank, + ST_X(p.geometry) AS lon, ST_Y(p.geometry) AS lat, + p.effective_at, p.uploaded_at, p.compass_angle, + p.width, p.height, p.original_filename, p.title, p.description, + p.sizes, u.username, p.owner_id + FROM panoramax.sequence_photos sp + JOIN photos p ON p.id = sp.photo_id + JOIN users u ON u.id = p.owner_id + WHERE sp.sequence_id = :seq_id AND sp.rank > :after_rank AND {ELIGIBLE_PHOTO_WHERE} + ORDER BY sp.rank + LIMIT :limit +""" + + +@app.get('/api/collections/{seq_id}/items') +async def items( + request: Request, + seq_id: str, + limit: int = Query(default=settings.ITEMS_PAGE_DEFAULT, ge=1, + le=settings.ITEMS_PAGE_MAX), + page_after_rank: int = Query(default=0, ge=0), +) -> JSONResponse: + scope = settings.active_scope() + row = await _load_sequence(seq_id) + if row is None: + raise HTTPException(status_code=404, detail='Collection not found') + if row[1] == 'deleted': + raise HTTPException(status_code=404, detail='Collection is deleted') + + async with get_engine().connect() as conn: + photo_rows = (await conn.execute(text(_ITEMS_SQL), { + 'seq_id': row[0], + 'after_rank': page_after_rank, + 'limit': limit, + 'scope_legal_rights': scope.legal_rights, + })).all() + + base = _base_url() + features = [] + for r in photo_rows: + feature = item_json( + photo_id=r[0], seq_id=seq_id, rank=r[1], + lon=r[2], lat=r[3], + effective_at=r[4], uploaded_at=r[5], + compass_angle=r[6], width=r[7], height=r[8], + original_filename=r[9], title=r[10], description=r[11], + sizes=r[12] or {}, username=r[13], owner_id=r[14], + license_id=scope.license, base_url=base, + ) + if feature: + features.append(feature) + + self_href = f"{base}/api/collections/{seq_id}/items" + links = [ + {'rel': 'self', 'href': str(request.url), 'type': 'application/geo+json'}, + {'rel': 'collection', 'href': f"{base}/api/collections/{seq_id}", 'type': 'application/json'}, + {'rel': 'root', 'href': f"{base}/api/", 'type': 'application/json'}, + ] + if len(photo_rows) == limit: + next_params = dict(request.query_params) + next_params['page_after_rank'] = str(photo_rows[-1][1]) + links.append({ + 'rel': 'next', + 'href': f"{self_href}?{urlencode(next_params)}", + 'type': 'application/geo+json', + }) + return JSONResponse({ + 'type': 'FeatureCollection', + 'features': features, + 'links': links, + }) + + +@app.get('/api/collections/{seq_id}/items/{item_id}') +async def item(seq_id: str, item_id: str) -> dict: + scope = settings.active_scope() + row = await _load_sequence(seq_id) + if row is None or row[1] == 'deleted': + raise HTTPException(status_code=404, detail='Collection not found') + + async with get_engine().connect() as conn: + r = (await conn.execute(text(f""" + SELECT + p.id, sp.rank, + ST_X(p.geometry) AS lon, ST_Y(p.geometry) AS lat, + p.effective_at, p.uploaded_at, p.compass_angle, + p.width, p.height, p.original_filename, p.title, p.description, + p.sizes, u.username, p.owner_id + FROM panoramax.sequence_photos sp + JOIN photos p ON p.id = sp.photo_id + JOIN users u ON u.id = p.owner_id + WHERE sp.sequence_id = :seq_id AND p.id = :item_id AND {ELIGIBLE_PHOTO_WHERE} + """), { + 'seq_id': row[0], 'item_id': item_id, + 'scope_legal_rights': scope.legal_rights, + })).first() + if r is None: + raise HTTPException(status_code=404, detail='Item not found') + + feature = item_json( + photo_id=r[0], seq_id=seq_id, rank=r[1], + lon=r[2], lat=r[3], + effective_at=r[4], uploaded_at=r[5], + compass_angle=r[6], width=r[7], height=r[8], + original_filename=r[9], title=r[10], description=r[11], + sizes=r[12] or {}, username=r[13], owner_id=r[14], + license_id=scope.license, base_url=_base_url(), + ) + if feature is None: + raise HTTPException(status_code=404, detail='Item not found') + return feature diff --git a/backend/panoramax/app/sequencer.py b/backend/panoramax/app/sequencer.py new file mode 100644 index 00000000..e443a808 --- /dev/null +++ b/backend/panoramax/app/sequencer.py @@ -0,0 +1,258 @@ +"""Sequence synthesis: split each owner's eligible photos into time-gap sessions +and persist them as panoramax.sequences / sequence_photos. + +Why synthesized persisted sequences (not one ever-growing collection per user): +the meta-catalog's incremental sync is collection-level — any change re-fetches +ALL items of the collection — so giant collections are an unbounded recurring +harvest cost. Sessions split on a capture-time gap (~3h default, env-tunable). +No distance split: the catalog only draws lines between consecutive items <75m +apart, so sparse sequences just render as dots. + +The run is a full deterministic recompute diffed against stored state, applying +only actual changes (so sequences.updated_at — the harvester's crawl signal, +bumped by the membership triggers — moves only when membership really changed). +Photos have no updated-at column to drive a cheaper incremental pass, and the +diff makes the recompute idempotent anyway. Stability rules: + +- A session keeps the UUID of the existing sequence it overlaps most (greedy, + larger overlap first; ties break on session order). Brand-new sessions get + fresh UUIDs — real UUIDs, the catalog casts collection ids to UUID PKs. +- A sequence whose photos all left (deleted / hidden / license flip / gap + merge) loses its membership rows; the membership trigger tombstones it + (status='deleted'). Tombstones are never hard-deleted and revive if the + sequencer repopulates them. +""" +import argparse +import asyncio +import logging +import uuid +from dataclasses import dataclass +from datetime import datetime, timedelta + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncEngine + +from eligibility import ELIGIBLE_PHOTO_WHERE, PHOTO_ORDER_BY +from settings import Scope, active_scope, sequencer_interval_s, session_gap_hours + +logger = logging.getLogger('panoramax.sequencer') + + +@dataclass(frozen=True) +class PhotoStub: + id: str + owner_id: str + effective_at: datetime + + +@dataclass(frozen=True) +class Membership: + photo_id: str + sequence_id: str + rank: int + + +def split_sessions(photos: list[PhotoStub], gap: timedelta) -> list[list[PhotoStub]]: + """Split an owner's capture-ordered photos wherever consecutive effective_at + differ by more than `gap`.""" + sessions: list[list[PhotoStub]] = [] + current: list[PhotoStub] = [] + prev_at: datetime | None = None + for photo in photos: + if prev_at is not None and photo.effective_at - prev_at > gap: + sessions.append(current) + current = [] + current.append(photo) + prev_at = photo.effective_at + if current: + sessions.append(current) + return sessions + + +def assign_sequence_ids( + sessions: list[list[PhotoStub]], + existing_seq_of_photo: dict[str, str], +) -> list[tuple[str, list[PhotoStub]]]: + """Give each session a stable sequence UUID. + + Overlap counting is against current membership; the greedy pass hands each + existing sequence to the single session overlapping it most, so a session + that swallowed two sequences (a gap closed) keeps the bigger one's identity + and the other tombstones. + """ + overlaps: list[tuple[int, int, str]] = [] # (overlap, session_idx, seq_id) + for idx, session in enumerate(sessions): + counts: dict[str, int] = {} + for photo in session: + seq = existing_seq_of_photo.get(photo.id) + if seq: + counts[seq] = counts.get(seq, 0) + 1 + for seq, count in counts.items(): + overlaps.append((count, idx, seq)) + # larger overlap first; deterministic tie-break on (session order, seq id) + overlaps.sort(key=lambda t: (-t[0], t[1], t[2])) + + assigned: dict[int, str] = {} + used_seqs: set[str] = set() + for count, idx, seq in overlaps: + if idx in assigned or seq in used_seqs: + continue + assigned[idx] = seq + used_seqs.add(seq) + + result = [] + for idx, session in enumerate(sessions): + seq_id = assigned.get(idx) or str(uuid.uuid4()) + result.append((seq_id, session)) + return result + + +def desired_memberships( + assigned_sessions: list[tuple[str, list[PhotoStub]]], +) -> list[Membership]: + out = [] + for seq_id, session in assigned_sessions: + for rank, photo in enumerate(session, start=1): + out.append(Membership(photo_id=photo.id, sequence_id=seq_id, rank=rank)) + return out + + +def diff_memberships( + current: list[Membership], desired: list[Membership] +) -> tuple[list[Membership], list[Membership], list[str]]: + """-> (to_insert, to_update, photo_ids_to_delete). Only actual changes, so + an unchanged owner produces zero writes and zero updated_at churn.""" + current_by_photo = {m.photo_id: m for m in current} + desired_by_photo = {m.photo_id: m for m in desired} + to_insert = [m for pid, m in desired_by_photo.items() if pid not in current_by_photo] + to_update = [ + m for pid, m in desired_by_photo.items() + if pid in current_by_photo and current_by_photo[pid] != m + ] + to_delete = [pid for pid in current_by_photo if pid not in desired_by_photo] + return to_insert, to_update, to_delete + + +async def run_once(engine: AsyncEngine, scope: Scope | None = None, gap: timedelta | None = None) -> dict: + """One full synthesis pass. Returns counters for logging/tests.""" + scope = scope or active_scope() + gap = gap or timedelta(hours=session_gap_hours()) + + async with engine.begin() as conn: + rows = (await conn.execute(text(f""" + SELECT p.id, p.owner_id, p.effective_at + FROM photos p + JOIN users u ON u.id = p.owner_id + WHERE {ELIGIBLE_PHOTO_WHERE} + ORDER BY p.owner_id, {PHOTO_ORDER_BY} + """), {'scope_legal_rights': scope.legal_rights})).all() + photos = [PhotoStub(id=r[0], owner_id=r[1], effective_at=r[2]) for r in rows] + + rows = (await conn.execute(text(""" + SELECT sp.photo_id, sp.sequence_id, sp.rank, s.owner_id + FROM panoramax.sequence_photos sp + JOIN panoramax.sequences s ON s.id = sp.sequence_id + WHERE s.scope = :scope + """), {'scope': scope.id})).all() + current = [Membership(photo_id=r[0], sequence_id=str(r[1]), rank=r[2]) for r in rows] + + existing_seq_ids = {m.sequence_id for m in current} + existing_seq_of_photo = {m.photo_id: m.sequence_id for m in current} + + # per-owner sessions -> globally desired memberships + desired: list[Membership] = [] + new_sequences: list[tuple[str, str]] = [] # (seq_id, owner_id) + by_owner: dict[str, list[PhotoStub]] = {} + for photo in photos: + by_owner.setdefault(photo.owner_id, []).append(photo) + for owner_id, owner_photos in by_owner.items(): + sessions = split_sessions(owner_photos, gap) + assigned = assign_sequence_ids(sessions, existing_seq_of_photo) + for seq_id, session in assigned: + if seq_id not in existing_seq_ids: + new_sequences.append((seq_id, owner_id)) + desired.extend(desired_memberships(assigned)) + + to_insert, to_update, to_delete = diff_memberships(current, desired) + + for seq_id, owner_id in new_sequences: + await conn.execute(text(""" + INSERT INTO panoramax.sequences (id, scope, status, owner_id) + VALUES (:id, :scope, 'ready', :owner_id) + """), {'id': uuid.UUID(seq_id), 'scope': scope.id, 'owner_id': owner_id}) + + # rank-unique is DEFERRABLE INITIALLY DEFERRED, so delete/update/insert + # order inside this transaction can't collide transiently + if to_delete: + await conn.execute( + text("DELETE FROM panoramax.sequence_photos WHERE photo_id = ANY(:pids)"), + {'pids': to_delete}) + for m in to_update: + await conn.execute(text(""" + UPDATE panoramax.sequence_photos + SET sequence_id = :seq, rank = :rank + WHERE photo_id = :pid + """), {'seq': uuid.UUID(m.sequence_id), 'rank': m.rank, 'pid': m.photo_id}) + for m in to_insert: + await conn.execute(text(""" + INSERT INTO panoramax.sequence_photos (photo_id, sequence_id, rank) + VALUES (:pid, :seq, :rank) + """), {'pid': m.photo_id, 'seq': uuid.UUID(m.sequence_id), 'rank': m.rank}) + + counters = { + 'eligible_photos': len(photos), + 'sequences_created': len(new_sequences), + 'memberships_inserted': len(to_insert), + 'memberships_updated': len(to_update), + 'memberships_deleted': len(to_delete), + } + if any(v for k, v in counters.items() if k != 'eligible_photos'): + logger.info("sequencer pass: %s", counters) + else: + logger.debug("sequencer pass: no changes (%d eligible photos)", len(photos)) + return counters + + +async def wait_for_schema(engine: AsyncEngine) -> None: + """The api container applies migration 030 in its prestart; this container + may win the race, so poll instead of crashing.""" + while True: + try: + async with engine.connect() as conn: + present = (await conn.execute( + text("SELECT to_regclass('panoramax.sequences')"))).scalar() + if present: + return + logger.warning("panoramax schema not migrated yet, waiting…") + except Exception as e: + logger.warning("database not reachable yet (%s), waiting…", e) + await asyncio.sleep(5) + + +async def loop(engine: AsyncEngine) -> None: + await wait_for_schema(engine) + interval = sequencer_interval_s() + while True: + try: + await run_once(engine) + except Exception: + logger.exception("sequencer pass failed") + await asyncio.sleep(interval) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Panoramax sequence synthesizer") + parser.add_argument('--once', action='store_true', help="run a single pass and exit") + args = parser.parse_args() + logging.basicConfig(level=logging.INFO) + + from db import get_engine + engine = get_engine() + if args.once: + asyncio.run(run_once(engine)) + else: + asyncio.run(loop(engine)) + + +if __name__ == '__main__': + main() diff --git a/backend/panoramax/app/settings.py b/backend/panoramax/app/settings.py new file mode 100644 index 00000000..a703e8a7 --- /dev/null +++ b/backend/panoramax/app/settings.py @@ -0,0 +1,90 @@ +"""Configuration for the Panoramax-compatible read API. + +The instance serves exactly one *scope* — a partition of Hillview's photos by +license. The federation's meta-catalog accepts only CC-BY-SA-4.0 / Licence +Ouverte 2.0 instances (enforced by human review at registration, never by +code), so the single scope declared instance-wide is the CC-BY-SA one. The +scope object keeps code/data modular: a second "ARR + OSM mapper provision" +instance later is a new Scope entry + deployment, not a config framework. +""" +import os +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Scope: + id: str + # photos.legal_rights value that selects this scope's photos + legal_rights: str + # SPDX id declared instance-wide and on every collection/item + license: str + license_url: str + + +SCOPES = { + 'cc': Scope( + id='cc', + legal_rights='ccbysa4+osm', + license='CC-BY-SA-4.0', + license_url='https://creativecommons.org/licenses/by-sa/4.0/', + ), +} + + +def active_scope() -> Scope: + return SCOPES[os.getenv('PANORAMAX_SCOPE', 'cc')] + + +def base_url() -> str: + """Public canonical base URL of this instance (no trailing slash, no /api). + + Registered in the meta-catalog verbatim. Caveat on the registered string: + the catalog's canonical_url() does a char-class rstrip("/api"), so a URL + ending in any of '/', 'a', 'p', 'i' loses those characters. `.cz` ends in + 'z', so every *.hillview.cz host is safe (and a trailing slash just gets + stripped, which is the intent) — but a path-suffixed URL would need care + (".../cc" is fine, ".../cc-osm-map" would be mangled to ".../cc-osm-m"). + + Default host is cc.geovisio.hillview.cz: this service speaks the GeoVisio + STAC dialect rather than being a Panoramax instance, and the `cc.` prefix + leaves room for a second license scope on its own host later. It also + keeps panoramax.hillview.cz free for an actual Panoramax deployment. + """ + return os.getenv('PANORAMAX_BASE_URL', 'http://localhost:8058').rstrip('/') + + +def viewer_url() -> str: + """Where a human landing on this service's root should go. + + The meta-catalog attaches a `rel=via` link to every harvested item whose + href is the registered instance URL, presented as "Link to the original + instance" — i.e. it is a viewer link, not an API link. Serving raw STAC + JSON there would be a dead end, so `/` redirects here. + """ + return os.getenv('PANORAMAX_VIEWER_URL', 'https://hillview.cz').rstrip('/') + + +def instance_name() -> str: + # Unique key in the meta-catalog's `instances` table (the URL is not + # unique, the name is). + return os.getenv('PANORAMAX_INSTANCE_NAME', 'Hillview') + + +def session_gap_hours() -> float: + return float(os.getenv('PANORAMAX_SESSION_GAP_HOURS', '3')) + + +def sequencer_interval_s() -> int: + return int(os.getenv('PANORAMAX_SEQUENCER_INTERVAL_S', '300')) + + +def sequencer_enabled() -> bool: + return os.getenv('PANORAMAX_SEQUENCER_ENABLED', 'true').lower() in ('1', 'true', 'yes') + + +# Page sizes. The harvester follows rel=next links, so limits only shape page +# count, not completeness. +COLLECTIONS_PAGE_DEFAULT = 100 +COLLECTIONS_PAGE_MAX = 1000 +ITEMS_PAGE_DEFAULT = 100 +ITEMS_PAGE_MAX = 1000 diff --git a/backend/panoramax/app/stac.py b/backend/panoramax/app/stac.py new file mode 100644 index 00000000..6a4f0e1d --- /dev/null +++ b/backend/panoramax/app/stac.py @@ -0,0 +1,249 @@ +"""STAC serialization: sequences -> Collections, photos -> Items. + +Shapes follow what the meta-catalog harvester actually consumes (verified +against its schema and code): +- collection/item ids are cast to UUID PK columns in the catalog; +- items get `collection` (UUID), NOT NULL GeoJSON `geometry`, and a + `properties.datetime` postgres can parse; +- collection `providers[*].id` is cast to a UUID global PK (we use the owner's + user id) and `name` is NOT NULL; +- tombstones are flagged by top-level `geovisio:status: "deleted"` while the + CQL2 `status` queryable filters them — both names refer to the same state; +- ordering is `properties["geovisio:rank_in_collection"]` (int, 1-based); +- absolute asset hrefs are kept verbatim by the harvester, so we point straight + at the existing CDN/pics WebP derivatives. +""" +import json +from datetime import datetime, timezone +from typing import Any + +STAC_VERSION = '1.0.0' + + +def fmt_dt(dt: datetime | None) -> str | None: + """ISO-8601 with microseconds and a numeric UTC offset — NEVER a 'Z' + suffix: the meta-catalog's jsonb_date() SQL function (indexes + incremental + crawl) parses collection created/updated with the hard-coded format + YYYY-MM-DD"T"HH24:MI:SS.USTZH:TZM, which rejects 'Z' and needs the .US + microseconds present. Naive datetimes in this DB are UTC by convention + (photos.captured_at/effective_at).""" + if dt is None: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc).isoformat(timespec='microseconds') + + +def _numeric_variants(sizes: dict[str, Any]) -> dict[int, dict]: + """The plain downscale variants ('320', '2048', ...) — excludes 'full', + '*_crop' and '*_llm' keys. JSON object keys are always strings even though + the worker builds them as ints.""" + out = {} + for key, info in sizes.items(): + if isinstance(info, dict) and str(key).isdigit() and info.get('url'): + out[int(key)] = info + return out + + +def pick_assets(sizes: dict[str, Any]) -> dict[str, Any] | None: + """Map Hillview's sizes JSON to GeoVisio's hd/sd/thumb assets. + + hd -> 'full', sd -> 2048-ish, thumb -> 640-ish. Fast-mode processing has no + 640 variant and narrow sources may lack larger ones, hence fallback chains. + All derivatives are WebP (risk noted in docs: the ecosystem tends to assume + jpeg; the fallback plan is jpeg derivatives, not serving different URLs). + Returns None when no usable asset exists (photo shouldn't be served then). + """ + if isinstance(sizes, str): + # photos.sizes is json (not jsonb); depending on driver codec setup a + # raw-SQL fetch may hand it over undecoded + try: + sizes = json.loads(sizes) + except ValueError: + return None + if not isinstance(sizes, dict): + return None + numeric = _numeric_variants(sizes) + full = sizes.get('full') if isinstance(sizes.get('full'), dict) else None + if full and not full.get('url'): + full = None + + def variant_at_most(cap: int) -> dict | None: + widths = [w for w in numeric if w <= cap] + return numeric[max(widths)] if widths else None + + def smallest() -> dict | None: + return numeric[min(numeric)] if numeric else None + + hd = full or (numeric[max(numeric)] if numeric else None) + if hd is None: + return None + sd = variant_at_most(2048) or smallest() or hd + thumb = (numeric.get(640) or numeric.get(320) or smallest() or hd) + + def asset(info: dict, title: str, roles: list[str]) -> dict: + a = { + 'href': info['url'], + 'type': 'image/webp', + 'title': title, + 'roles': roles, + } + if info.get('width'): + a['width'] = info['width'] + if info.get('height'): + a['height'] = info['height'] + return a + + return { + 'hd': asset(hd, 'HD picture', ['data']), + 'sd': asset(sd, 'SD picture', ['visual']), + 'thumb': asset(thumb, 'Thumbnail', ['thumbnail']), + } + + +def provider_name(username: str | None, owner_id: str | None) -> str: + # providers.name is NOT NULL in the catalog + if username: + return username + if owner_id: + return f"user-{owner_id[:8]}" + return 'unknown' + + +def collection_json( + *, + seq_id: str, + status: str, + owner_id: str | None, + username: str | None, + created_at: datetime, + updated_at: datetime, + item_count: int, + bbox: list[float] | None, + min_dt: datetime | None, + max_dt: datetime | None, + license_id: str, + license_url: str, + base_url: str, +) -> dict[str, Any]: + self_href = f"{base_url}/api/collections/{seq_id}" + if status == 'deleted': + # Tombstone: served forever so the harvester's incremental diff learns + # about the deletion. Minimal on purpose — the harvester only needs id + # + geovisio:status + updated. + return { + 'type': 'Collection', + 'stac_version': STAC_VERSION, + 'id': seq_id, + 'geovisio:status': 'deleted', + 'created': fmt_dt(created_at), + 'updated': fmt_dt(updated_at), + 'license': license_id, + 'extent': { + 'spatial': {'bbox': [[-180.0, -90.0, 180.0, 90.0]]}, + 'temporal': {'interval': [[None, None]]}, + }, + 'description': 'Deleted sequence', + 'links': [ + {'rel': 'self', 'href': self_href, 'type': 'application/json'}, + ], + } + + name = provider_name(username, owner_id) + title = f"Photos by {name}" + (f" — {fmt_dt(min_dt)[:10]}" if min_dt else '') + spatial_bbox = [bbox] if bbox else [[-180.0, -90.0, 180.0, 90.0]] + return { + 'type': 'Collection', + 'stac_version': STAC_VERSION, + 'id': seq_id, + 'title': title, + 'description': f"Sequence of photos captured by {name} on Hillview", + 'geovisio:status': 'ready', + 'license': license_id, + 'created': fmt_dt(created_at), + 'updated': fmt_dt(updated_at), + 'keywords': ['pictures'], + 'providers': [ + # id is a UUID global PK in the catalog: the owner's user id. + # Tombstoned-owner sequences (owner_id NULL) never reach this + # branch with items, but guard anyway. + {'name': name, 'roles': ['producer'], **({'id': owner_id} if owner_id else {})}, + ], + 'extent': { + 'spatial': {'bbox': spatial_bbox}, + 'temporal': {'interval': [[fmt_dt(min_dt), fmt_dt(max_dt)]]}, + }, + 'stats:items': {'count': item_count}, + 'links': [ + {'rel': 'self', 'href': self_href, 'type': 'application/json'}, + {'rel': 'root', 'href': f"{base_url}/api/", 'type': 'application/json'}, + {'rel': 'parent', 'href': f"{base_url}/api/", 'type': 'application/json'}, + {'rel': 'items', 'href': f"{self_href}/items", 'type': 'application/geo+json'}, + {'rel': 'license', 'href': license_url, 'title': license_id}, + ], + } + + +def item_json( + *, + photo_id: str, + seq_id: str, + rank: int, + lon: float, + lat: float, + effective_at: datetime, + uploaded_at: datetime | None, + compass_angle: float | None, + width: int | None, + height: int | None, + original_filename: str | None, + title: str | None, + description: str | None, + sizes: dict[str, Any], + username: str | None, + owner_id: str | None, + license_id: str, + base_url: str, +) -> dict[str, Any] | None: + assets = pick_assets(sizes) + if assets is None: + return None + name = provider_name(username, owner_id) + properties: dict[str, Any] = { + 'datetime': fmt_dt(effective_at), + 'created': fmt_dt(uploaded_at), + 'license': license_id, + 'geovisio:status': 'ready', + 'geovisio:rank_in_collection': rank, + 'geovisio:producer': name, + # All Hillview photos are flat — no pano capture path exists. + 'pers:interior_orientation': {}, + } + if compass_angle is not None: + properties['view:azimuth'] = round(compass_angle) % 360 + if original_filename: + properties['original_file:name'] = original_filename + if title: + properties['title'] = title + if description: + properties['description'] = description + + collection_href = f"{base_url}/api/collections/{seq_id}" + return { + 'type': 'Feature', + 'stac_version': STAC_VERSION, + 'id': photo_id, + 'collection': seq_id, + 'geometry': {'type': 'Point', 'coordinates': [lon, lat]}, + 'bbox': [lon, lat, lon, lat], + 'properties': properties, + 'providers': [ + {'name': name, 'roles': ['producer'], **({'id': owner_id} if owner_id else {})}, + ], + 'assets': assets, + 'links': [ + {'rel': 'self', 'href': f"{collection_href}/items/{photo_id}", 'type': 'application/geo+json'}, + {'rel': 'collection', 'href': collection_href, 'type': 'application/json'}, + {'rel': 'root', 'href': f"{base_url}/api/", 'type': 'application/json'}, + ], + } diff --git a/backend/panoramax/app/tests/unit/test_cql.py b/backend/panoramax/app/tests/unit/test_cql.py new file mode 100644 index 00000000..632c6e43 --- /dev/null +++ b/backend/panoramax/app/tests/unit/test_cql.py @@ -0,0 +1,112 @@ +"""The CQL2-text subset parser must accept exactly what the meta-catalog +harvester sends (see its harvest.py get_collections) and reject everything +else loudly.""" +from datetime import datetime, timezone + +import pytest + +from cql import CollectionsFilter, FilterParseError, parse_collections_filter + + +class TestHarvesterShapes: + def test_no_filter_full_harvest(self): + assert parse_collections_filter(None) == CollectionsFilter() + assert parse_collections_filter('') == CollectionsFilter() + assert parse_collections_filter(' ') == CollectionsFilter() + + def test_exact_incremental_filter(self): + # verbatim shape from harvest.py:67 + f = parse_collections_filter( + "status IN ('deleted','ready') AND updated > '2026-07-01T12:34:56Z'") + assert f.statuses == {'deleted', 'ready'} + assert f.updated_after == datetime(2026, 7, 1, 12, 34, 56, tzinfo=timezone.utc) + assert f.updated_inclusive is False + + def test_timestamp_with_offset(self): + f = parse_collections_filter("updated > '2026-07-01T12:00:00+02:00'") + assert f.updated_after == datetime(2026, 7, 1, 10, 0, tzinfo=timezone.utc) + + def test_naive_timestamp_assumed_utc(self): + f = parse_collections_filter("updated > '2026-07-01T12:00:00'") + assert f.updated_after.tzinfo == timezone.utc + + +class TestGrammarTolerance: + def test_clauses_in_reverse_order(self): + f = parse_collections_filter( + "updated > '2026-01-01T00:00:00Z' AND status IN ('ready')") + assert f.statuses == {'ready'} + assert f.updated_after is not None + + def test_case_insensitive_keywords(self): + f = parse_collections_filter( + "STATUS in ('ready','deleted') and UPDATED > '2026-01-01T00:00:00Z'") + assert f.statuses == {'ready', 'deleted'} + + def test_status_equality(self): + assert parse_collections_filter("status = 'deleted'").statuses == {'deleted'} + assert parse_collections_filter("status='ready'").statuses == {'ready'} + + def test_updated_gte_is_inclusive(self): + f = parse_collections_filter("updated >= '2026-01-01T00:00:00Z'") + assert f.updated_inclusive is True + + def test_spaces_inside_in_list(self): + f = parse_collections_filter("status IN ( 'ready' , 'deleted' )") + assert f.statuses == {'ready', 'deleted'} + + def test_cql2_timestamp_literal(self): + # proper CQL2 spelling, which the harvester doesn't use but a spec- + # following client would + f = parse_collections_filter("updated > TIMESTAMP('2026-07-01T12:00:00Z')") + assert f.updated_after == datetime(2026, 7, 1, 12, 0, tzinfo=timezone.utc) + + def test_parenthesised_clause(self): + f = parse_collections_filter( + "status IN ('ready') AND (updated >= '2026-01-01T00:00:00Z')") + assert f.statuses == {'ready'} + assert f.updated_inclusive is True + + def test_quoted_identifier(self): + assert parse_collections_filter("\"status\" = 'deleted'").statuses == {'deleted'} + + def test_three_clauses_left_nested(self): + # pygeofilter nests `a AND b AND c` as And(And(a, b), c); the inner And + # must be flattened, not rejected as an unknown clause — which shows up + # as the third clause reaching the duplicate check + with pytest.raises(FilterParseError, match='duplicate status'): + parse_collections_filter( + "status IN ('ready') AND updated > '2026-01-01T00:00:00Z' AND status IN ('deleted')") + + +class TestRejection: + @pytest.mark.parametrize('bad', [ + "garbage", + "status IN (ready)", # unquoted literal → Attribute node + "status IN ('bogus')", # unknown status + "status IN ()", # grammar error + "updated > 2026-01-01", # unquoted → arithmetic 2026-1-1 + "updated > 'not-a-date'", + "updated > DATE('2026-01-01')", # date literal has no time part + "updated > 20260101", # numeric literal + "updated < '2026-01-01T00:00:00Z'", # unsupported operator + "created > '2026-01-01T00:00:00Z'", # unsupported field + "status = 'ready' OR status = 'deleted'", # OR not in the subset + "NOT status = 'ready'", + "status NOT IN ('ready')", + "status IS NULL", + "status LIKE 'read%'", + "'ready' = status", # attribute must be the lhs + "status = 'ready' AND status = 'deleted'", # duplicate clause + "updated > '2026-01-01T00:00:00Z' AND updated > '2026-01-02T00:00:00Z'", + "status = 'ready' AND (updated > '2026-01-01T00:00:00Z' OR status = 'deleted')", + ]) + def test_rejects(self, bad): + with pytest.raises(FilterParseError): + parse_collections_filter(bad) + + def test_error_message_names_the_offender(self): + with pytest.raises(FilterParseError, match='created'): + parse_collections_filter("created > '2026-01-01T00:00:00Z'") + with pytest.raises(FilterParseError, match='NOT IN'): + parse_collections_filter("status NOT IN ('ready')") diff --git a/backend/panoramax/app/tests/unit/test_sequencer_logic.py b/backend/panoramax/app/tests/unit/test_sequencer_logic.py new file mode 100644 index 00000000..8729c045 --- /dev/null +++ b/backend/panoramax/app/tests/unit/test_sequencer_logic.py @@ -0,0 +1,140 @@ +"""Pure-logic tests for session splitting, sequence identity assignment and +membership diffing — the sequencer's correctness core, no DB involved.""" +import uuid +from datetime import datetime, timedelta + +from sequencer import ( + Membership, + PhotoStub, + assign_sequence_ids, + desired_memberships, + diff_memberships, + split_sessions, +) + +GAP = timedelta(hours=3) +T0 = datetime(2026, 7, 1, 8, 0, 0) + + +def stub(i: int, at: datetime, owner: str = 'owner1') -> PhotoStub: + return PhotoStub(id=f'photo-{i}', owner_id=owner, effective_at=at) + + +class TestSplitSessions: + def test_empty(self): + assert split_sessions([], GAP) == [] + + def test_single_photo(self): + photos = [stub(1, T0)] + assert split_sessions(photos, GAP) == [photos] + + def test_no_split_within_gap(self): + photos = [stub(i, T0 + timedelta(hours=i)) for i in range(4)] + assert split_sessions(photos, GAP) == [photos] + + def test_split_beyond_gap(self): + morning = [stub(1, T0), stub(2, T0 + timedelta(minutes=10))] + evening = [stub(3, T0 + timedelta(hours=9)), stub(4, T0 + timedelta(hours=9, minutes=5))] + assert split_sessions(morning + evening, GAP) == [morning, evening] + + def test_exactly_gap_does_not_split(self): + photos = [stub(1, T0), stub(2, T0 + GAP)] + assert split_sessions(photos, GAP) == [photos] + + def test_just_over_gap_splits(self): + photos = [stub(1, T0), stub(2, T0 + GAP + timedelta(seconds=1))] + assert split_sessions(photos, GAP) == [[photos[0]], [photos[1]]] + + def test_multiple_splits(self): + days = [[stub(10 * d + i, T0 + timedelta(days=d, minutes=i)) for i in range(2)] + for d in range(3)] + flat = [p for day in days for p in day] + assert split_sessions(flat, GAP) == days + + +class TestAssignSequenceIds: + def test_new_sessions_get_fresh_valid_uuids(self): + sessions = split_sessions([stub(1, T0), stub(2, T0 + timedelta(hours=9))], GAP) + assigned = assign_sequence_ids(sessions, {}) + assert len(assigned) == 2 + ids = [seq_id for seq_id, _ in assigned] + assert len(set(ids)) == 2 + for seq_id in ids: + uuid.UUID(seq_id) # must be real UUIDs — catalog casts to UUID PKs + + def test_unchanged_session_keeps_sequence_id(self): + photos = [stub(1, T0), stub(2, T0 + timedelta(minutes=5))] + existing = {p.id: 'seq-A' for p in photos} + assigned = assign_sequence_ids([photos], existing) + assert assigned == [('seq-A', photos)] + + def test_growing_session_keeps_sequence_id(self): + old = [stub(1, T0), stub(2, T0 + timedelta(minutes=5))] + new_photo = stub(3, T0 + timedelta(minutes=10)) + existing = {p.id: 'seq-A' for p in old} + assigned = assign_sequence_ids([old + [new_photo]], existing) + assert assigned[0][0] == 'seq-A' + + def test_gap_close_merge_keeps_bigger_sequence(self): + # two sequences whose photos now fall into one session (e.g. gap config + # raised): the one contributing more photos keeps its identity + a = [stub(i, T0 + timedelta(minutes=i)) for i in range(3)] + b = [stub(10 + i, T0 + timedelta(hours=1, minutes=i)) for i in range(2)] + existing = {p.id: 'seq-A' for p in a} | {p.id: 'seq-B' for p in b} + assigned = assign_sequence_ids([a + b], existing) + assert assigned[0][0] == 'seq-A' + + def test_session_split_bigger_part_keeps_id(self): + # a sequence split in two (photos removed in the middle): the larger + # fragment keeps the id, the smaller gets a fresh one + big = [stub(i, T0 + timedelta(minutes=i)) for i in range(3)] + small = [stub(10, T0 + timedelta(hours=9))] + existing = {p.id: 'seq-A' for p in big + small} + assigned = assign_sequence_ids([big, small], existing) + assert assigned[0][0] == 'seq-A' + assert assigned[1][0] != 'seq-A' + uuid.UUID(assigned[1][0]) + + def test_equal_overlap_tie_breaks_on_session_order(self): + a = [stub(1, T0)] + b = [stub(2, T0 + timedelta(hours=9))] + existing = {'photo-1': 'seq-A', 'photo-2': 'seq-A'} + assigned = assign_sequence_ids([a, b], existing) + # seq-A goes to the first session; second gets a new id + assert assigned[0][0] == 'seq-A' + assert assigned[1][0] != 'seq-A' + + +class TestDiffMemberships: + def m(self, pid: str, seq: str, rank: int) -> Membership: + return Membership(photo_id=pid, sequence_id=seq, rank=rank) + + def test_no_change(self): + cur = [self.m('p1', 's1', 1), self.m('p2', 's1', 2)] + ins, upd, dele = diff_memberships(cur, list(cur)) + assert (ins, upd, dele) == ([], [], []) + + def test_insert_update_delete(self): + cur = [self.m('p1', 's1', 1), self.m('p2', 's1', 2), self.m('p3', 's1', 3)] + # p2 gone -> p3 moves up, p4 appended + desired = [self.m('p1', 's1', 1), self.m('p3', 's1', 2), self.m('p4', 's1', 3)] + ins, upd, dele = diff_memberships(cur, desired) + assert ins == [self.m('p4', 's1', 3)] + assert upd == [self.m('p3', 's1', 2)] + assert dele == ['p2'] + + def test_sequence_move_is_update(self): + cur = [self.m('p1', 's1', 1)] + desired = [self.m('p1', 's2', 1)] + ins, upd, dele = diff_memberships(cur, desired) + assert (ins, dele) == ([], []) + assert upd == [self.m('p1', 's2', 1)] + + +class TestDesiredMemberships: + def test_ranks_are_one_based_per_sequence(self): + s1 = [stub(1, T0), stub(2, T0 + timedelta(minutes=1))] + s2 = [stub(3, T0 + timedelta(hours=9))] + out = desired_memberships([('seq-A', s1), ('seq-B', s2)]) + assert [(m.sequence_id, m.rank) for m in out] == [ + ('seq-A', 1), ('seq-A', 2), ('seq-B', 1)] diff --git a/backend/panoramax/app/tests/unit/test_stac.py b/backend/panoramax/app/tests/unit/test_stac.py new file mode 100644 index 00000000..f8ec23bf --- /dev/null +++ b/backend/panoramax/app/tests/unit/test_stac.py @@ -0,0 +1,207 @@ +"""Serialization tests: asset fallback chains against real worker size layouts, +tombstone shape, datetime formatting (microseconds + numeric offset — the +meta-catalog's jsonb_date() rejects 'Z').""" +import json +from datetime import datetime, timezone + +from stac import collection_json, fmt_dt, item_json, pick_assets, provider_name + +# Worker layouts (photo_processor.create_optimized_sizes): JSON object keys are +# strings after persistence. Fast mode has no 640. +FULL_SIZES = { + 'full': {'url': 'https://pics/full.webp', 'width': 4000, 'height': 3000}, + '320': {'url': 'https://pics/320.webp', 'width': 320, 'height': 240}, + '640': {'url': 'https://pics/640.webp', 'width': 640, 'height': 480}, + '1200': {'url': 'https://pics/1200.webp', 'width': 1200, 'height': 900}, + '2048': {'url': 'https://pics/2048.webp', 'width': 2048, 'height': 1536}, + '3072': {'url': 'https://pics/3072.webp', 'width': 3072, 'height': 2304}, + '320_crop': {'url': 'https://pics/320c.webp', 'width': 320, 'height': 240}, + '1200_crop': {'url': 'https://pics/1200c.webp', 'width': 1200, 'height': 630}, + '640_llm': {'url': 'https://pics/640llm.webp', 'width': 640, 'height': 480}, +} +FAST_SIZES = { + 'full': {'url': 'https://pics/full.webp', 'width': 4000, 'height': 3000}, + '320': {'url': 'https://pics/320.webp', 'width': 320, 'height': 240}, + '1200': {'url': 'https://pics/1200.webp', 'width': 1200, 'height': 900}, + '2048': {'url': 'https://pics/2048.webp', 'width': 2048, 'height': 1536}, +} + + +class TestPickAssets: + def test_full_layout(self): + a = pick_assets(FULL_SIZES) + assert a['hd']['href'] == 'https://pics/full.webp' + assert a['sd']['href'] == 'https://pics/2048.webp' + assert a['thumb']['href'] == 'https://pics/640.webp' + assert all(v['type'] == 'image/webp' for v in a.values()) + assert a['hd']['roles'] == ['data'] + assert a['sd']['roles'] == ['visual'] + assert a['thumb']['roles'] == ['thumbnail'] + + def test_fast_mode_thumb_falls_back_to_320(self): + a = pick_assets(FAST_SIZES) + assert a['thumb']['href'] == 'https://pics/320.webp' + assert a['sd']['href'] == 'https://pics/2048.webp' + + def test_crop_and_llm_variants_never_used(self): + a = pick_assets(FULL_SIZES) + hrefs = json.dumps(a) + assert 'crop' not in hrefs and 'llm' not in hrefs + + def test_narrow_source_only_full_and_320(self): + sizes = {k: FULL_SIZES[k] for k in ('full', '320')} + a = pick_assets(sizes) + assert a['hd']['href'] == 'https://pics/full.webp' + assert a['sd']['href'] == 'https://pics/320.webp' + assert a['thumb']['href'] == 'https://pics/320.webp' + + def test_no_full_uses_largest_numeric_as_hd(self): + sizes = {k: FULL_SIZES[k] for k in ('320', '2048')} + assert pick_assets(sizes)['hd']['href'] == 'https://pics/2048.webp' + + def test_json_string_input(self): + assert pick_assets(json.dumps(FULL_SIZES))['hd']['href'] == 'https://pics/full.webp' + + def test_unusable_inputs(self): + assert pick_assets({}) is None + assert pick_assets(None) is None + assert pick_assets('not json') is None + assert pick_assets({'full': {'path': 'x'}}) is None # no url + + def test_width_height_carried(self): + a = pick_assets(FULL_SIZES) + assert (a['thumb']['width'], a['thumb']['height']) == (640, 480) + + +class TestFmtDt: + def test_naive_is_utc_with_offset(self): + assert fmt_dt(datetime(2026, 7, 1, 12, 0, 5)) == '2026-07-01T12:00:05.000000+00:00' + + def test_aware_converted_to_utc(self): + dt = datetime(2026, 7, 1, 14, 0, tzinfo=timezone.utc) + assert fmt_dt(dt) == '2026-07-01T14:00:00.000000+00:00' + + def test_none(self): + assert fmt_dt(None) is None + + +class TestProviderName: + def test_prefers_username(self): + assert provider_name('alice', 'uuid-1') == 'alice' + + def test_fallback_is_never_null(self): + # providers.name is NOT NULL in the catalog + assert provider_name(None, 'abcdef12-3456') == 'user-abcdef12' + assert provider_name(None, None) == 'unknown' + + +BASE_ITEM_KWARGS = dict( + photo_id='11111111-2222-3333-4444-555555555555', + seq_id='aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee', + rank=3, + lon=14.42, lat=50.09, + effective_at=datetime(2026, 7, 1, 10, 30), + uploaded_at=datetime(2026, 7, 2, 8, 0, tzinfo=timezone.utc), + compass_angle=283.6, + width=4000, height=3000, + original_filename='IMG_1234.jpg', + title='A hill', description='A view of a hill', + sizes=FULL_SIZES, + username='alice', owner_id='99999999-8888-7777-6666-555555555555', + license_id='CC-BY-SA-4.0', + base_url='https://panoramax.hillview.cz', +) + + +class TestItemJson: + def test_core_fields(self): + item = item_json(**BASE_ITEM_KWARGS) + assert item['type'] == 'Feature' + assert item['id'] == BASE_ITEM_KWARGS['photo_id'] + assert item['collection'] == BASE_ITEM_KWARGS['seq_id'] + assert item['geometry'] == {'type': 'Point', 'coordinates': [14.42, 50.09]} + assert item['bbox'] == [14.42, 50.09, 14.42, 50.09] + + def test_properties(self): + p = item_json(**BASE_ITEM_KWARGS)['properties'] + assert p['datetime'] == '2026-07-01T10:30:00.000000+00:00' + assert p['created'] == '2026-07-02T08:00:00.000000+00:00' + assert p['geovisio:rank_in_collection'] == 3 + assert p['view:azimuth'] == 284 + assert p['license'] == 'CC-BY-SA-4.0' + assert p['geovisio:producer'] == 'alice' + assert p['original_file:name'] == 'IMG_1234.jpg' + + def test_azimuth_wraps_and_optional(self): + item = item_json(**{**BASE_ITEM_KWARGS, 'compass_angle': 359.7}) + assert item['properties']['view:azimuth'] == 0 + item = item_json(**{**BASE_ITEM_KWARGS, 'compass_angle': None}) + assert 'view:azimuth' not in item['properties'] + + def test_provider_id_present(self): + providers = item_json(**BASE_ITEM_KWARGS)['providers'] + assert providers == [{ + 'name': 'alice', 'roles': ['producer'], + 'id': BASE_ITEM_KWARGS['owner_id'], + }] + + def test_unservable_sizes_returns_none(self): + assert item_json(**{**BASE_ITEM_KWARGS, 'sizes': {}}) is None + + def test_self_link_under_collection(self): + links = {l['rel']: l['href'] for l in item_json(**BASE_ITEM_KWARGS)['links']} + assert links['self'].endswith( + '/api/collections/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/items/11111111-2222-3333-4444-555555555555') + + +BASE_COLLECTION_KWARGS = dict( + seq_id='aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee', + status='ready', + owner_id='99999999-8888-7777-6666-555555555555', + username='alice', + created_at=datetime(2026, 7, 1, tzinfo=timezone.utc), + updated_at=datetime(2026, 7, 3, tzinfo=timezone.utc), + item_count=12, + bbox=[14.4, 50.0, 14.5, 50.1], + min_dt=datetime(2026, 7, 1, 10, 0), + max_dt=datetime(2026, 7, 1, 11, 0), + license_id='CC-BY-SA-4.0', + license_url='https://creativecommons.org/licenses/by-sa/4.0/', + base_url='https://panoramax.hillview.cz', +) + + +class TestCollectionJson: + def test_ready_collection(self): + c = collection_json(**BASE_COLLECTION_KWARGS) + assert c['type'] == 'Collection' + assert c['geovisio:status'] == 'ready' + assert c['license'] == 'CC-BY-SA-4.0' + assert c['stats:items'] == {'count': 12} + assert c['extent']['spatial']['bbox'] == [[14.4, 50.0, 14.5, 50.1]] + assert c['extent']['temporal']['interval'] == [['2026-07-01T10:00:00.000000+00:00', '2026-07-01T11:00:00.000000+00:00']] + assert c['providers'][0]['id'] == BASE_COLLECTION_KWARGS['owner_id'] + assert c['created'] == '2026-07-01T00:00:00.000000+00:00' + assert c['updated'] == '2026-07-03T00:00:00.000000+00:00' + + def test_self_link(self): + c = collection_json(**BASE_COLLECTION_KWARGS) + links = {l['rel']: l['href'] for l in c['links']} + # the harvester fetches items via "/items" + assert links['self'] == 'https://panoramax.hillview.cz/api/collections/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee' + assert links['items'] == links['self'] + '/items' + + def test_tombstone(self): + c = collection_json(**{ + **BASE_COLLECTION_KWARGS, + 'status': 'deleted', 'item_count': 0, 'bbox': None, + 'min_dt': None, 'max_dt': None, 'owner_id': None, 'username': None, + }) + assert c['geovisio:status'] == 'deleted' + assert c['id'] == BASE_COLLECTION_KWARGS['seq_id'] + assert c['updated'] == '2026-07-03T00:00:00.000000+00:00' + # tombstones still parse as a Collection (id/type/stac_version/extent) + assert c['type'] == 'Collection' + assert 'extent' in c + # but must not leak owner info + assert 'providers' not in c diff --git a/backend/panoramax/pyproject.toml b/backend/panoramax/pyproject.toml new file mode 100644 index 00000000..b254d254 --- /dev/null +++ b/backend/panoramax/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "hillview-panoramax" +version = "1.0.0" +description = "Panoramax/GeoVisio-compatible read API serving Hillview's CC-licensed photos to the Panoramax federation" +requires-python = ">=3.12" + +dependencies = [ + "fastapi", + "uvicorn[standard]", + "sqlalchemy[asyncio]", + "asyncpg", + # real CQL2-text grammar for the /api/collections `filter` param; same + # library the reference GeoVisio server uses (it pins ~=0.3.1) + "pygeofilter>=0.3.1", +] + +[project.optional-dependencies] +dev = [ + "pytest==9.0.3", + "pytest-asyncio>=1.0,<2", + "httpx", +] diff --git a/backend/panoramax/run_unit_tests.sh b/backend/panoramax/run_unit_tests.sh new file mode 100755 index 00000000..9ec0e39c --- /dev/null +++ b/backend/panoramax/run_unit_tests.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Unit tests for the panoramax federation service (no DB needed). +# Mirrors backend/api/run_unit_tests.sh. +set -e + +cd "$(dirname "$(readlink -f -- "$0")")/.." # backend/ + +uv sync --quiet --frozen --package hillview-panoramax --all-extras + +cd panoramax/app +export PYTHONPATH="$(pwd):$(pwd)/../.." + +if [ $# -eq 0 ]; then + uv run --quiet pytest tests/unit/ -v +else + uv run --quiet pytest "$@" +fi diff --git a/backend/panoramax/scripts/e2e_federation.sh b/backend/panoramax/scripts/e2e_federation.sh new file mode 100755 index 00000000..2d406929 --- /dev/null +++ b/backend/panoramax/scripts/e2e_federation.sh @@ -0,0 +1,207 @@ +#!/bin/bash +# End-to-end federation test: drive the REAL Panoramax meta-catalog harvester +# against this repo's panoramax container and assert that every contract the +# federation depends on actually holds. +# +# ./backend/panoramax/scripts/e2e_federation.sh # full run +# ./backend/panoramax/scripts/e2e_federation.sh --no-seed # use existing photos +# ./backend/panoramax/scripts/e2e_federation.sh --keep-up # don't stop the catalog after +# ./backend/panoramax/scripts/e2e_federation.sh --down # tear the catalog down and exit +# +# What it exercises, in order: +# 1. hillview stack up (postgres+api+worker+panoramax), migration applied, +# panoramax_ro provisioned +# 2. seeds CC-licensed photos through the real upload path, laid out as N +# time-gap sessions -> asserts the sequencer produces exactly N sequences +# 3. meta-catalog stack up, harvester installed, instance registered +# 4. full harvest: asserts collections + items land with ZERO harvest errors +# (a single bad datetime format silently zeroes the whole import) +# 5. photo edit -> incremental harvest re-fetches exactly that collection +# 6. soft-delete -> sequencer prune -> incremental harvest moves the item to +# deleted_items; emptying a sequence tombstones it and the tombstone stays +# listable via the CQL status filter +# 7. restore -> everything comes back +# 8. pystac validation of a served collection + item +# +# Requires: the meta-catalog checked out at $META_CATALOG (default below), uv, +# docker. Leaves the hillview stack running; stops the catalog stack unless +# --keep-up. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$(readlink -f -- "$0")")/../../.." && pwd)" +META_CATALOG="${META_CATALOG:-/home/koom/repos/panoramax/server/meta-catalog/0/meta-catalog}" +WORK_DIR="${WORK_DIR:-${TMPDIR:-/tmp}/panoramax-e2e}" +CATALOG_DB='postgresql://username:password@localhost:5439/panoramax' +INSTANCE_NAME="${INSTANCE_NAME:-hillview-e2e}" +PANORAMAX_URL="${PANORAMAX_URL:-http://localhost:8058}" +SESSIONS="${SESSIONS:-3}" +PER_SESSION="${PER_SESSION:-4}" + +SEED=1 +KEEP_UP=0 +for arg in "$@"; do + case "$arg" in + --no-seed) SEED=0 ;; + --keep-up) KEEP_UP=1 ;; + --down) docker compose -f "$META_CATALOG/docker-compose.yml" down; exit 0 ;; + *) echo "unknown option: $arg" >&2; exit 2 ;; + esac +done + +RED=$'\e[31m'; GREEN=$'\e[32m'; BOLD=$'\e[1m'; RESET=$'\e[0m' +step() { echo; echo "${BOLD}==> $*${RESET}"; } +ok() { echo "${GREEN} ✓ $*${RESET}"; } +fail() { echo "${RED} ✗ $*${RESET}" >&2; exit 1; } + +assert_eq() { # assert_eq + [ "$1" = "$2" ] && ok "$3 = $1" || fail "$3: expected $2, got $1" +} + +hv_psql() { docker exec hillview_postgres psql -U "${POSTGRES_USER:-hillview}" -d "${POSTGRES_DB:-hillview}" -tA -c "$1"; } +cat_psql() { docker exec meta-catalog-database-1 psql -U username -d panoramax -tA -c "$1"; } +harvester() { "$WORK_DIR/venv/bin/stac-harvester" "$@" --db "$CATALOG_DB"; } +sequencer() { docker exec hillview_panoramax python /app/app/sequencer.py --once 2>&1 | tail -1; } + +# Collections whose harvest failed. This is the check that matters most: the +# harvester logs errors per collection and still reports "imported", so a +# schema mismatch looks like success until you count the errors. +harvest_errors_since() { cat_psql "SELECT count(*) FROM harvest_errors he JOIN harvests h ON h.id = he.harvest_id WHERE h.start > now() - interval '$1'"; } + +cd "$REPO_ROOT" +[ -f .env ] || fail ".env missing — copy one in before running (see .env.example)" +set -a; . ./.env; set +a +[ -n "${PANORAMAX_DB_PASSWORD:-}" ] || fail "PANORAMAX_DB_PASSWORD unset in .env" +[ -d "$META_CATALOG" ] || fail "meta-catalog not found at $META_CATALOG (set META_CATALOG=)" + +step "1/8 hillview stack" +./compose.sh --profile panoramax up --build -d postgres api worker panoramax >/dev/null +until curl -sf "$PANORAMAX_URL/api/health" >/dev/null 2>&1; do sleep 2; done +ok "panoramax API healthy at $PANORAMAX_URL" +[ "$(hv_psql "SELECT count(*) FROM alembic_version WHERE version_num = '030_add_panoramax_schema'")" = "1" ] \ + || fail "migration 030 not applied (api prestart should have done it)" +ok "migration 030 applied" +./backend/scripts/provision_panoramax_role.sh >/dev/null 2>&1 && ok "panoramax_ro provisioned" +until curl -sf "http://localhost:8055/api/debug" >/dev/null 2>&1; do sleep 2; done +ok "hillview api reachable" + +if [ "$SEED" = "1" ]; then + step "2/8 seed $SESSIONS sessions × $PER_SESSION photos through the real upload path" + mkdir -p "$WORK_DIR" + (cd backend && uv run --quiet --frozen --package hillview-tests \ + python panoramax/scripts/seed_photos.py \ + --sessions "$SESSIONS" --per-session "$PER_SESSION" \ + --out "$WORK_DIR/photo_ids.txt") + SEEDED_USER=$(hv_psql "SELECT owner_id FROM photos WHERE id = '$(head -1 "$WORK_DIR/photo_ids.txt")'") + # The seeder uploads as the debug test user, but eligibility deliberately + # excludes users.is_test (we don't federate test accounts). Clear the flag + # on the seeded user so the rest of the run exercises the real path — + # `./backend/debug.sh recreate` restores it. + hv_psql "UPDATE users SET is_test = false WHERE id = '$SEEDED_USER'" >/dev/null + ok "seeded user $SEEDED_USER un-flagged as test (eligibility excludes is_test)" + sequencer + assert_eq "$(hv_psql "SELECT count(*) FROM panoramax.sequences WHERE owner_id = '$SEEDED_USER' AND status = 'ready'")" \ + "$SESSIONS" "sequences synthesized for the seeded user" +else + step "2/8 seeding skipped (--no-seed)" + sequencer +fi + +TOTAL_SEQ=$(hv_psql "SELECT count(*) FROM panoramax.sequences WHERE status = 'ready'") +TOTAL_PHOTOS=$(hv_psql "SELECT count(*) FROM panoramax.sequence_photos") +ok "$TOTAL_SEQ ready sequences / $TOTAL_PHOTOS memberships" + +step "3/8 meta-catalog stack + harvester" +docker compose -f "$META_CATALOG/docker-compose.yml" up -d database migrations >/dev/null 2>&1 +until docker exec meta-catalog-database-1 pg_isready -U username -d panoramax >/dev/null 2>&1; do sleep 2; done +until [ "$(cat_psql "SELECT to_regclass('public.instances') IS NOT NULL")" = "t" ]; do sleep 2; done +ok "catalog database migrated" +if [ ! -x "$WORK_DIR/venv/bin/stac-harvester" ]; then + mkdir -p "$WORK_DIR" + uv venv "$WORK_DIR/venv" --quiet + VIRTUAL_ENV="$WORK_DIR/venv" uv pip install --quiet -e "$META_CATALOG/harvester" 'pystac[validation]' +fi +ok "harvester CLI ready" + +# Re-register from scratch so each run is independent of catalog leftovers +cat_psql "DELETE FROM instances WHERE name = '$INSTANCE_NAME'" >/dev/null +harvester add-instance "$INSTANCE_NAME" --url "$PANORAMAX_URL" 2>&1 | grep -q "added with id" \ + && ok "instance registered (configuration endpoint accepted)" \ + || fail "add-instance failed — /api/configuration is mandatory, check it returns 200" + +step "4/8 full harvest" +harvester harvest "$INSTANCE_NAME" --full-harvest 2>&1 | grep -E "🎉|Collections imported: [0-9]+ col \[0" | tail -1 +assert_eq "$(harvest_errors_since '5 minutes')" "0" "harvest errors" +assert_eq "$(cat_psql "SELECT count(*) FROM collections")" "$TOTAL_SEQ" "collections in catalog" +assert_eq "$(cat_psql "SELECT count(*) FROM items")" "$TOTAL_PHOTOS" "items in catalog" +[ "$(cat_psql "SELECT count(*) FROM providers")" -gt 0 ] && ok "providers derived" || fail "no providers — providers[*].id/name missing?" +[ "$(cat_psql "SELECT count(*) FROM collections WHERE computed_geom IS NOT NULL")" -gt 0 ] \ + && ok "collection geometries computed (items ordered by rank)" \ + || fail "no computed_geom — geovisio:rank_in_collection or geometry is wrong" + +step "5/8 edit propagation (incremental)" +read -r SEQ PHOTO <<<"$(hv_psql "SELECT sp.sequence_id, sp.photo_id FROM panoramax.sequence_photos sp JOIN (SELECT sequence_id, count(*) c FROM panoramax.sequence_photos GROUP BY 1 HAVING count(*) > 1 ORDER BY c LIMIT 1) s ON s.sequence_id = sp.sequence_id ORDER BY sp.rank LIMIT 1" | tr '|' ' ')" +[ -n "$PHOTO" ] || fail "no multi-photo sequence to test with" +MARK="e2e-$(hv_psql "SELECT floor(extract(epoch from now()))::bigint")" +hv_psql "UPDATE photos SET title = '$MARK' WHERE id = '$PHOTO'" >/dev/null +harvester harvest "$INSTANCE_NAME" --incremental-harvest >/dev/null 2>&1 +assert_eq "$(cat_psql "SELECT content->'properties'->>'title' FROM items WHERE id = '$PHOTO'")" "$MARK" "edited title in catalog" +assert_eq "$(harvest_errors_since '2 minutes')" "0" "harvest errors after edit" + +step "6/8 deletion propagation + tombstones" +hv_psql "UPDATE photos SET deleted = true WHERE id = '$PHOTO'" >/dev/null +curl -sf "$PANORAMAX_URL/api/collections/$SEQ/items?limit=1000" \ + | grep -q "\"$PHOTO\"" && fail "serve-time filter did not hide the deleted photo" \ + || ok "deleted photo gone from /items immediately (before the sequencer ran)" +sequencer +harvester harvest "$INSTANCE_NAME" --incremental-harvest >/dev/null 2>&1 +assert_eq "$(cat_psql "SELECT count(*) FROM items WHERE id = '$PHOTO'")" "0" "deleted item removed from catalog" +assert_eq "$(cat_psql "SELECT count(*) FROM deleted_items WHERE id = '$PHOTO'")" "1" "item recorded in deleted_items" + +# empty a whole sequence -> tombstone, still listable via the CQL status filter +LONE_SEQ=$(hv_psql "SELECT sequence_id FROM panoramax.sequence_photos GROUP BY 1 HAVING count(*) = 1 LIMIT 1") +if [ -n "$LONE_SEQ" ]; then + LONE_PHOTO=$(hv_psql "SELECT photo_id FROM panoramax.sequence_photos WHERE sequence_id = '$LONE_SEQ'") + hv_psql "UPDATE photos SET deleted = true WHERE id = '$LONE_PHOTO'" >/dev/null + sequencer + assert_eq "$(hv_psql "SELECT status FROM panoramax.sequences WHERE id = '$LONE_SEQ'")" "deleted" "emptied sequence status" + curl -sf "$PANORAMAX_URL/api/collections?limit=1000" | grep -q "$LONE_SEQ" \ + && fail "tombstone leaked into the default (unfiltered) collections listing" \ + || ok "tombstone hidden from default listing" + # limit=1000: the listing is paginated (ordered by id, 100 per page), so + # without it this check only sees page one and passes or fails depending + # on where the tombstone's UUID happens to sort + curl -sf -G "$PANORAMAX_URL/api/collections" \ + --data-urlencode "filter=status IN ('deleted','ready') AND updated > '2000-01-01T00:00:00.000000+00:00'" \ + --data-urlencode "limit=1000" \ + | grep -q "$LONE_SEQ" && ok "tombstone served via the harvester's CQL filter" \ + || fail "tombstone NOT served via the status filter — deletions would never propagate" + harvester harvest "$INSTANCE_NAME" --incremental-harvest >/dev/null 2>&1 + assert_eq "$(cat_psql "SELECT count(*) FROM collections WHERE id = '$LONE_SEQ'")" "0" "tombstoned collection dropped by the catalog" + hv_psql "UPDATE photos SET deleted = false WHERE id = '$LONE_PHOTO'" >/dev/null +fi + +step "7/8 restore" +hv_psql "UPDATE photos SET deleted = false, title = NULL WHERE id = '$PHOTO'" >/dev/null +sequencer +harvester harvest "$INSTANCE_NAME" --incremental-harvest >/dev/null 2>&1 +assert_eq "$(cat_psql "SELECT count(*) FROM items WHERE id = '$PHOTO'")" "1" "restored item back in catalog" +assert_eq "$(cat_psql "SELECT count(*) FROM items")" "$(hv_psql "SELECT count(*) FROM panoramax.sequence_photos")" "catalog items == memberships after churn" + +step "8/8 pystac validation" +"$WORK_DIR/venv/bin/python" - "$PANORAMAX_URL" <<'EOF' +import json, sys, urllib.request +import pystac +base = sys.argv[1] +col = json.load(urllib.request.urlopen(f'{base}/api/collections?limit=1'))['collections'][0] +pystac.Collection.from_dict(col).validate() +item = json.load(urllib.request.urlopen(f"{base}/api/collections/{col['id']}/items?limit=1"))['features'][0] +pystac.Item.from_dict(item).validate() +print(' \033[32m✓ collection + item validate against the STAC schemas\033[0m') +EOF + +if [ "$KEEP_UP" = "0" ]; then + docker compose -f "$META_CATALOG/docker-compose.yml" down >/dev/null 2>&1 + echo; echo "${GREEN}${BOLD}e2e federation test PASSED${RESET} (catalog stack stopped; --keep-up to leave it running)" +else + echo; echo "${GREEN}${BOLD}e2e federation test PASSED${RESET} (catalog left running: db :5439, harvester in $WORK_DIR/venv)" +fi diff --git a/backend/panoramax/scripts/seed_photos.py b/backend/panoramax/scripts/seed_photos.py new file mode 100644 index 00000000..16db11b3 --- /dev/null +++ b/backend/panoramax/scripts/seed_photos.py @@ -0,0 +1,106 @@ +"""Upload CC-licensed test photos through the real upload path, shaped so the +sequencer has something interesting to split. + +Used by e2e_federation.sh. Goes through the full authorize → worker → process +flow (SecureUploadClient), so the photos end up with real derivatives in +`sizes` — which is what the federation API serves as assets. + +Photos are laid out as N sessions of M photos: within a session, captures are +minutes apart; between sessions, a gap wider than PANORAMAX_SESSION_GAP_HOURS. +So a successful run must produce exactly N sequences for the test user. +""" +import argparse +import asyncio +import os +import sys +from datetime import datetime, timedelta, timezone + +BACKEND = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) +sys.path.insert(0, os.path.join(BACKEND, 'tests')) +sys.path.insert(0, BACKEND) + +from utils.image_utils import create_test_image_full_gps # noqa: E402 +from utils.secure_upload_utils import SecureUploadClient # noqa: E402 +from utils.test_utils import wait_for_photo_processing # noqa: E402 + +API_URL = os.getenv('API_URL', 'http://localhost:8055/api') +# The API hands clients the deployment's public WORKER_URL (e.g. a Caddy vhost +# that only resolves inside the VM). For a local e2e run we talk to the worker +# directly instead of routing through whatever the deployment advertises. +WORKER_URL = os.getenv('WORKER_URL_E2E', 'http://localhost:8056') + +# Prague-ish, walked west→east so consecutive photos are ~40 m apart (under the +# catalog's 75 m line-drawing threshold, so harvested sequences render as lines) +BASE_LAT, BASE_LON = 50.0755, 14.4378 +STEP_LON = 0.00055 + + +async def seed(sessions: int, per_session: int, gap_hours: float) -> list[str]: + client = SecureUploadClient(api_url=API_URL) + setup = await client.setup_test_environment() + token = await client.test_user_auth(setup) + keys = client.generate_client_keys() + await client.register_client_key(token, keys) + + start = datetime.now(timezone.utc) - timedelta(days=2) + photo_ids: list[str] = [] + n = 0 + for s in range(sessions): + session_start = start + timedelta(hours=s * (gap_hours + 1)) + for i in range(per_session): + captured = session_start + timedelta(minutes=2 * i) + lat = BASE_LAT + lon = BASE_LON + STEP_LON * n + bearing = (90 + 5 * i) % 360 + image = create_test_image_full_gps(240, 180, (40 * s % 255, 90, 160), lat, lon, bearing) + filename = f"panoramax_e2e_s{s}_p{i}.jpg" + auth = await client.authorize_upload_with_params( + auth_token=token, + filename=filename, + file_size=len(image), + latitude=lat, + longitude=lon, + description=f"panoramax e2e session {s} photo {i}", + file_data=image, + captured_at=captured.isoformat(), + license='ccbysa4+osm', + title=f"E2E session {s} #{i}", + ) + auth['worker_url'] = WORKER_URL + await client.upload_to_worker(image, auth, keys, filename=filename) + photo_ids.append(auth['photo_id']) + n += 1 + print(f" uploaded {filename} -> {auth['photo_id']}") + + print(f"waiting for processing of {len(photo_ids)} photos…") + failed = [] + for pid in photo_ids: + photo = wait_for_photo_processing(pid, token, timeout=120) + if photo.get('processing_status') != 'completed': + failed.append((pid, photo.get('processing_status'), photo.get('error'))) + if failed: + for pid, status, err in failed: + print(f" FAILED {pid}: {status} ({err})", file=sys.stderr) + raise SystemExit(f"{len(failed)}/{len(photo_ids)} photos did not process") + + print(f"seeded {len(photo_ids)} photos in {sessions} sessions") + return photo_ids + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--sessions', type=int, default=3) + p.add_argument('--per-session', type=int, default=4) + p.add_argument('--gap-hours', type=float, + default=float(os.getenv('PANORAMAX_SESSION_GAP_HOURS', '3'))) + p.add_argument('--out', help="write the uploaded photo ids here, one per line") + args = p.parse_args() + + ids = asyncio.run(seed(args.sessions, args.per_session, args.gap_hours)) + if args.out: + with open(args.out, 'w') as f: + f.write('\n'.join(ids) + '\n') + + +if __name__ == '__main__': + main() diff --git a/backend/pyproject.toml b/backend/pyproject.toml index d9024bdb..fa3364d7 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -13,7 +13,7 @@ requires-python = ">=3.12" exclude-newer = "2026-08-11T00:00:00Z" [tool.uv.workspace] -members = ["common", "api/app", "worker", "tests"] +members = ["common", "api/app", "worker", "tests", "panoramax"] [tool.ruff] exclude = [ diff --git a/backend/scripts/provision_panoramax_role.sh b/backend/scripts/provision_panoramax_role.sh new file mode 100755 index 00000000..6d03bdd5 --- /dev/null +++ b/backend/scripts/provision_panoramax_role.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Provision the panoramax_ro role on an EXISTING deployment (initdb.d only runs +# on fresh clusters). Idempotent: safe to re-run; re-applies grants. +# +# Usage (from repo root, with .env loaded or POSTGRES_* + PANORAMAX_DB_PASSWORD +# exported): +# ./backend/scripts/provision_panoramax_role.sh +# +# Must run AFTER migration 030_add_panoramax_schema (the grants reference the +# panoramax schema). Migration 030 also applies these grants itself when the +# role already exists, so on fresh clusters this script is unnecessary. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +if [ -f "$REPO_ROOT/.env" ]; then + set -a + # shellcheck disable=SC1091 + source "$REPO_ROOT/.env" + set +a +fi + +: "${POSTGRES_USER:?POSTGRES_USER is required}" +: "${POSTGRES_DB:?POSTGRES_DB is required}" +: "${PANORAMAX_DB_PASSWORD:?PANORAMAX_DB_PASSWORD is required}" + +CONTAINER="${POSTGRES_CONTAINER:-hillview_postgres}" + +# '' -escape any single quotes so the password can't break the SQL literal +ESCAPED_PW="${PANORAMAX_DB_PASSWORD//\'/\'\'}" + +docker exec -i "$CONTAINER" \ + psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" < "Open Data Licence" pictures are: +> - under CC-BY-SA 4.0 licence for every original or derivated picture +> - free to use (as in free speech) for creating other derivated data +> (including AI models) under Licence Ouverte 2.0, CC-BY 4.0 or ODbL 1.0 +> +> "Metadata and semantic tags" … are published under Licence Ouverte 2.0 and +> CC-BY 4.0. + +Hillview's `ccbysa4+osm` grants derived-data rights only for OSM +contributions, so on paper we cover the ODbL-via-OSM destination but not +LO 2.0 / CC-BY 4.0 generally. + +**In practice this is near-invisible.** The Panoramax config model allows +exactly one SPDX id + URL per instance, with no field for supplementary +permissions, and the meta-catalog performs no license validation whatsoever +(its own test fixtures carry `"license": "proprietary"`). So OSM-France's +extra grant lives only in its own prose; every consumer reading the federated +catalog sees bare `CC-BY-SA-4.0`. Measured against the live instances API, +**20 of 21 federated instances declare `CC-BY-SA-4.0`** (18 clean, 2 +malformed as `[CC-BY-SA-4.0]`) and one declares `etalab-2.0`. Join requests +consist of a bare licence line and are accepted as such; the project lead's +own guidance on the wider grant is phrased as advice ("I *suggest* to add the +authorization…"), not a condition. + +Conclusion: declaring `CC-BY-SA-4.0` and stating our narrower grant in the +registration issue and our own licensing page puts us in the same structural +position as almost every federated instance. Nothing in the harvester carries +the difference either way. + +**Undecided and worth deciding**: Hillview does not state a licence for photo +*metadata*, which is precisely what the catalog copies. OSM-France moved its +metadata and semantic tags to LO 2.0 + CC-BY 4.0 in February 2026. + +Where that bites, concretely (verified 2026-08-25): the federation itself +**redistributes the harvested metadata under Licence Ouverte 2.0**. The weekly +pg_dump + GeoParquet exports of api.panoramax.xyz are published on +data.gouv.fr as "Export du catalogue global des photos de Panoramax" +(organisation Panoramax, created 2025-11-24, licence `lov2`) — three months +*before* OSM-France relicensed its own metadata. Nothing in the catalog's +docs or API states this: the STAC landing page has no `license` field or +`rel=license` link, `/api/configuration` is a 404, the docs mention licences +only in the joining policy, and each item/collection merely carries the +*pictures* licence copied verbatim from its source instance (osm-fr items say +`CC-BY-SA-4.0`, ign says `etalab-2.0`). So the policy's "usable for creating +derivated data under LO 2.0 / CC-BY 4.0 / ODbL 1.0" clause is not advisory +after all: the catalog exercises it on every joined instance's metadata. An +instance that grants only CC-BY-SA-4.0 + an OSM-only grant (our current +`ccbysa4+osm`) has, strictly, not authorised that LO 2.0 export. Whatever we +decide for metadata should include an LO 2.0-compatible grant, or we join +knowing the export exceeds our terms as almost every other instance does. + +The scope machinery (`sequences.scope` column + `Scope` object in +`backend/panoramax/app/settings.py`) keeps a future second instance (e.g. "ARR ++ OSM mapper provision") a contained code change, not a config framework. + +A photo is **servable** iff (single definition in +`backend/panoramax/app/eligibility.py`, used by sequencer *and* API): +in-scope license, not soft-deleted, `is_public`, processing completed, has +geometry + effective_at + sizes, owner active and not a test user, no +thumbs-down rating, and no unresolved flag (moderation signals propagate to +the catalog on the sequencer's cadence; serve-time filtering hides the item +immediately). + +## Sequences + +Synthesized, persisted, per-owner **time-gap sessions** (default 3 h, env +`PANORAMAX_SESSION_GAP_HOURS`) — not one ever-growing collection per user, +because the meta-catalog's incremental sync re-fetches **all** items of any +changed collection (collection-level granularity), making giant collections an +unbounded recurring cost. No distance splitting: the catalog only draws lines +between consecutive items <75 m apart, so sparse sequences render as dots. + +Lifecycle invariants: + +- Sequence ids are real UUIDs (the catalog casts `content->>'id'` into UUID + primary keys — same for item ids, which are photo UUIDs). +- A sequence that loses all members flips to `status='deleted'` (trigger) and + is **never hard-deleted**: the harvester learns about deletions *only* by + listing `status IN ('deleted','ready') AND updated > ` — tombstones must + keep being served with a bumped `updated_at`. +- `owner_id` is `ON DELETE SET NULL` so tombstones survive account deletion + (membership rows cascade away; the membership trigger tombstones the + sequence in the same statement). +- Any relevant `photos` UPDATE (visibility, license, geometry, heading, + capture time, sizes, title/description, processing state, soft-delete) bumps + the owning sequence's `updated_at` via trigger — the only channel through + which member changes propagate to the catalog. Serve-time filtering makes + out-of-scope photos vanish from `/items` immediately, before the sequencer's + next pass prunes membership. +- The sequencer is a full deterministic recompute diffed against stored state + (photos have no updated-at column to drive anything cheaper); an unchanged + owner produces zero writes, so `updated_at` only moves on real change. + Session→sequence identity: keep the UUID of the sequence you overlap most. + +## Meta-catalog harvester contract (verified against its source) + +Repo: `gitlab.com/panoramax/server/meta-catalog` (local checkout: +`/home/koom/repos/panoramax/server/meta-catalog/0/meta-catalog`). + +- **Mandatory endpoints**: `/api/configuration` (`add-instance` aborts without + it; content stored verbatim, no fields inspected), `/api/collections` + (filter + `rel=next` paging), `/api/collections/{id}/items` (`limit` + + `rel=next`). Users/map/RSS/search are NOT consumed — the catalog regenerates + them. +- **Incremental sync** (~2 min cadence, ≥5 min per-instance floor) sends + exactly `?filter=status IN ('deleted','ready') AND updated > 'Z'` + (CQL2-text; `backend/panoramax/app/cql.py` parses it with pygeofilter — the + same grammar library the reference GeoVisio server uses — then accepts only + the `status`/`updated` conjunction we can execute, 400 on anything else), then + re-fetches ALL items of each changed collection; item deletions are detected + by diff. +- **Tombstone flag mismatch to be aware of**: the CQL2 queryable is `status`, + but the JSON field the harvester checks is `geovisio:status == "deleted"` + (harvest.py `sync_collection`). +- Items are ingested with `(content->>'id')::uuid`, `(content->>'collection')::uuid`, + `ST_GeomFromGeoJSON(content->'geometry')` **NOT NULL**, and + `properties.datetime` parsed by Postgres — all four must be present/valid. +- Collections need `id`, `created`, a `rel=self` link (items are fetched at + `/items`), and `providers[*]` with `{id: , name: + }`; we use the owner's user UUID + username. **Never share + provider or collection UUIDs across instances.** +- Ordering property: `properties["geovisio:rank_in_collection"]` (int, + 1-based). +- Instances are keyed by unique `name`; `url` carries no unique constraint, but + two instances sharing a URL would harvest identical data, so a second scope + needs its own URL (see the naming section). +- Asset hrefs that are absolute are kept verbatim → we point straight at the + existing pics/CDN **WebP** derivatives (`hd`→`full`, `sd`→2048-ish, + `thumb`→640-ish, with fallback chains for fast-mode/narrow photos — see + `stac.pick_assets`). See the WebP caveat below. + +### ⚠️ WebP: what actually breaks + +The viewer's compatibility spec lists `image/webp` alongside `image/jpeg`, and +the meta-catalog stores whatever asset types you declare, so WebP harvests +fine. The upstream history is worth knowing, because it is often misread as an +anti-WebP stance: WebP was the **primary storage format** in API 1.4.0 +(*"Internal storage format for pictures is now WebP, offering same quality with +reduced disk usage"*, with four `*_webp` STAC assets), was dropped from STAC in +2.0.0, and on-the-fly JPEG→WebP conversion was removed in 2.7.0 as *"too +slow"* — the changelog adding *"WebP might do an unexpected come-back in the +future 😉"*. The performance objection was therefore about **transcoding per +request**, a cost we do not have: every WebP we serve is a pre-generated static +file. Their reference ladder is `hd` = untouched original, `sd` = fixed 2048px +at quality 75, `thumb` = 500px at quality 75 (a 500×300 centre-crop for 360°), +tiles at 95; the thumbnail is a plain PIL resize, **not** an EXIF-embedded one +(there is no vips or exiftool anywhere in their API). + +We are nonetheless the only WebP producer in the federation. The federation +itself imposes no format requirement and the meta-catalog is format-agnostic +(it stores hrefs verbatim and 308-redirects), but **three places in the +official clients hardcode `image/jpeg`**: + +- `web-viewer` `API.js:471-475` — the thumbnail lookup requires JPEG for both + the `thumbnail` role and its `visual` fallback, so map hover popups get a + `null` URL and are silently removed. (Inconsistent with `picture.js:343`, + which *does* accept `image/webp` for the same `visual` role — the main photo + texture therefore displays fine.) +- `cli` `download.py:96` — filters assets on `type == "image/jpeg"` with no + `else`, so `panoramax-cli download` yields zero pictures from us, silently. +- viewer fast-mode preload — cosmetic only. + +Browsing via the federation is unaffected: the meta-catalog advertises its own +`item-preview`/`collection-preview` endpoints (typed `image/jpeg`, redirecting +to whatever we stored), and the viewer prefers those over per-item assets. The +breakage only hits a viewer pointed **directly** at our instance — which our +landing page currently guarantees, since it declares no `item-preview` link. + +Cheapest fixes, in order: declare `item-preview`/`collection-preview` links on +our landing page; or generate real JPEG for the `thumb` tier only (500px q75 ≈ +15 KiB/photo); or full JPEG derivatives for the CC subset (largest change — +the worker is WebP-only end to end, including `.webp` DZI tiles). + +Size-wise WebP is currently costing us rather than saving: at `WEBP_QUALITY_SIZES = 97` +our 2048px derivative is 304 KiB against 182 KiB for Panoramax's 2048px JPEG at +quality 75, and our 640px thumb is 47 KiB against their 15 KiB 500px thumb. The +format's ~25-34% advantage also lands at its low end for complex natural scenes. + +### ⚠️ Asset CORS + +`api.panoramax.xyz` fetches assets in the browser, so the asset host must send +permissive CORS — this has broken at least three instances right after they +joined. Current state: `pics.hillview.cz` sends +`Access-Control-Allow-Origin: *` ✅, but the **Tigris CDN +(`pics4.t3.storage.dev`) sends no CORS headers at all** ❌. No federated photo +lives there today (863 non-CC photos do), so it is a latent trap: configure +the bucket's CORS policy before any CC photo is written to that pool. + +## Deployment + +1. Set in `.env`: `PANORAMAX_DB_PASSWORD=`, and for prod + `PANORAMAX_BASE_URL=https://cc.geovisio.hillview.cz`. +2. Role: fresh clusters get `panoramax_ro` from initdb automatically; existing + deployments run `./backend/scripts/provision_panoramax_role.sh` (after the + api container has applied migration 030 — grants reference the schema). +3. `docker compose --profile panoramax up -d --build panoramax` (dev: + `./compose.sh --profile panoramax up -d --build panoramax`). +4. Caddy vhost (lives outside the repo, `~/caddy/Caddyfile` on the VM). See the + naming section for the `rstrip("/api")` caveat — any `*.hillview.cz` host is + safe: + + ```caddy + https://cc.geovisio.hillview.cz { + reverse_proxy localhost:8058 + } + ``` + +Env knobs: `PANORAMAX_SCOPE` (default `cc`), `PANORAMAX_SESSION_GAP_HOURS` +(3), `PANORAMAX_SEQUENCER_INTERVAL_S` (300), `PANORAMAX_SEQUENCER_ENABLED` +(true — set false to run the API without the in-process sequencer), +`PANORAMAX_INSTANCE_NAME` (Hillview), `PANORAMAX_VIEWER_URL` +(`https://hillview.cz` — where `/` redirects humans arriving from the +catalog's per-item `rel=via` link). + +## Registration + +A free-form issue on the meta-catalog GitLab project (no template exists): +instance URL, unique name, logo, whether the instance accepts external +contributions (ours: **no** — uploads go through hillview.cz), geographic +coverage, and a 24/7-availability *goal* (explicitly no SLA expected). +Approval is a manual decision by two maintainers, who then run +`stac-harvester add-instance`. Turnaround observed in past requests: same-day +to ~3 weeks. + +**We would be the first non-official server implementation in the federation.** +Every currently federated instance runs the official Panoramax API. The +project's stated position is standards-based (*"any compliant server can be a +part of Panoramax"*), and the viewer docs explicitly support third-party STAC +APIs — but there is no precedent, so expect scrutiny. The one existing +non-official implementation (PanoCommons, Wikimedia Commons imagery on +PgSTAC/stac-fastapi, built by a Panoramax maintainer) is *not* federated, and +would fail the harvester contract today: no `/api/configuration`, non-UUID +collection ids, no `created`/`updated`/`geovisio:status`. + +What reviewers have actually blocked or chased on past requests: + +- **Blurring** — faces and licence plates must be anonymised; one instance was + held up until it reprocessed its sequences. We comfortably exceed this: the + worker blurs *whole* persons and vehicles (`person`, `bicycle`, `car`, + `motorcycle`, `bus`, `truck` — `worker/detections.py`), and all 23,973 + currently-federated photos carry a detection record (6,400 with detected + objects), i.e. none were uploaded with anonymisation skipped. +- **`/api/configuration` completeness** — name, description, logo, contact, + colour, geographic coverage. Chased in nearly every thread. +- **Asset CORS** — see above. +- **URL scheme** — see the naming section. +- **Anonymous upload / open registration** — maintainers probe this (one + literally created a test collection on an applicant's instance). Ours is + read-only, so it is moot, but say so. +- **Verify the data actually lands.** Harvesting has silently failed to start + after approval at least twice due to catalog-side misconfiguration. After + being added, check `api.panoramax.xyz` for our collections rather than + assuming. + +Neighbours for reference: two Czech instances are already federated — +`pano.mahdi.cz` and `pano.kasik-gis.eu`, both single-user, CC-BY-SA-4.0, both +running the official Panoramax API. + +## Frontend self-duplicate filtering + +Once federated, api.panoramax.xyz serves our own photos back through the +`panoramax` source. `PanoramaxSourceLoader.isOwnInstanceItem()` drops items +whose `rel=via` link (added per-item by the catalog's `/api/search`, href = +origin instance URL) matches `ownPanoramaxInstanceUrls`, falling back to +asset-URL-prefix matching against `ownPhotoAssetUrlPrefixes` (pics hosts + +CDN) when no via link exists. Drops are counted like the hidden-content drops +(`droppedSelf`). Cross-source id-dedup is deliberately not the primary +mechanism: the hillview source caps photos per bbox, so the native copy may be +absent while the federated copy loads. + +## Verification plan + +Unit: `./backend/panoramax/run_unit_tests.sh` (CQL subset, session +splitting/identity/diffing, STAC serialization incl. asset fallbacks and +tombstones) and `frontend`: `bun run test:unit -- src/lib/sources/PanoramaxSourceLoader.test.ts`. + +End-to-end against the **real harvester**, fully scripted: + +```bash +./backend/panoramax/scripts/e2e_federation.sh # full run +./backend/panoramax/scripts/e2e_federation.sh --no-seed # reuse existing photos +./backend/panoramax/scripts/e2e_federation.sh --keep-up # leave the catalog running +./backend/panoramax/scripts/e2e_federation.sh --down # tear the catalog down +``` + +It brings up the hillview stack, seeds CC photos through the real upload path +(`scripts/seed_photos.py`, laid out as N time-gap sessions so the sequencer's +splitting is asserted), starts the meta-catalog stack from the local checkout +(`META_CATALOG=`, default `/home/koom/repos/panoramax/server/meta-catalog/0/meta-catalog`), +installs the harvester CLI, registers the instance, and then asserts: full +harvest with **zero harvest errors** (the check that matters — the harvester +reports "imported" even when every collection failed), collection/item counts +matching the DB, providers and computed geometries present, edit propagation +via incremental harvest, deletion propagation into `deleted_items`, sequence +tombstoning plus tombstone visibility through the CQL status filter, restore, +and pystac validation. + +Two things it does that are worth knowing: it points the seeder straight at +`localhost:8056` (the API advertises the deployment's public `WORKER_URL`, +which may not resolve locally), and it clears `users.is_test` on the seeded +user, because eligibility deliberately excludes test accounts. + +Running it in a fresh worktree needs three gitignored files copied from a +working checkout: `.env`, `backend/api/app/.env` (holds `JWT_PRIVATE_KEY` — +without it the API mints ephemeral keys and the worker 401s every upload), +`backend/worker/.env`, plus `backend/worker/models/` (~48 MB of YOLO weights, +needed to build the worker image). + +## State of play (2026-08-07) + +Built, tested and verified end-to-end against the real harvester; **not yet +committed** (branch `enrich`) and **not yet deployed or registered**. + +Verified working: migration 030 applies; 117 sequences / 23,961 memberships +synthesized from real dev data; full harvest of 120 collections / 23,973 items +with zero harvest errors; incremental harvest picks up exactly the +trigger-bumped collection; soft-delete propagates to `deleted_items`; emptied +sequences tombstone and stay listable through the CQL status filter; restore +round-trips; pystac validates both shapes; 59 backend + 26 frontend unit tests +pass. The `panoramax_ro` role was probed directly: `UPDATE`/`DELETE` on +`photos` and `users` are denied, writes succeed only inside the `panoramax` +schema. + +Open decisions, in rough priority order: + +1. **Metadata licence** — currently unstated (see the licensing section). +2. **WebP** — accept the two client breakages, add `item-preview` links, or + generate JPEG thumbs (see the WebP section). +3. **Prod deploy + registration** — Caddy vhost, then the GitLab issue. +4. **Tigris CORS** — before any CC photo is written to that pool. +5. Optionally lower `WEBP_QUALITY_SIZES` from 97; it is what makes our + derivatives heavier than Panoramax's JPEGs. diff --git a/enrich/db/seed_dev_from_dump.sh b/enrich/db/seed_dev_from_dump.sh index 810497c3..872075d1 100755 --- a/enrich/db/seed_dev_from_dump.sh +++ b/enrich/db/seed_dev_from_dump.sh @@ -17,16 +17,28 @@ # the workbench, which a truncate would destroy along with the round-trip they close. # The fresh path counts those and prints them before touching anything. # +# Fresh mode SNAPSHOTS BEFORE IT DESTROYS: the mirror only protects rows a sync has +# SEEN (Aug 2026: a 12-day sync gap meant 103 locally-created annotations were +# truncated without a trace), so a workbench reconcile runs before the truncate, and +# the post-reload sync then stamps whatever did not come back (missing_since — never +# deleted). Every workbench read filters stamped rows out, so test junk swept up this +# way is invisible dead weight, not pollution; the hygiene report at the end says +# what accumulated and which of it is purgeable. --no-presync skips the snapshot +# (and its workbench-must-be-up requirement) — that re-opens the blind spot, so +# prefer starting the workbench instead. +# # The truncate and the reload run in ONE transaction: a concurrent workbench reconcile # must never observe an empty source, or it would stamp every mirror row missing_since. set -euo pipefail MODE=fresh +PRESYNC=yes ARGS=() for a in "$@"; do case "$a" in - --additive) MODE=additive ;; - --fresh) MODE=fresh ;; + --additive) MODE=additive ;; + --fresh) MODE=fresh ;; + --no-presync) PRESYNC=no ;; -*) echo "unknown flag: $a" >&2; exit 2 ;; *) ARGS+=("$a") ;; esac @@ -34,13 +46,44 @@ done if [ "${#ARGS[@]}" -lt 2 ]; then # no defaults on purpose: the fresh path is destructive, and a default pointing at # whichever dump was newest when this was written is exactly the wrong footgun - echo "usage: $0 [--additive] " >&2 + echo "usage: $0 [--additive] [--no-presync] " >&2 exit 2 fi PHOTOS_CSV="${ARGS[0]}" ANNS_CSV="${ARGS[1]}" PG="psql -h 127.0.0.1 -p ${POSTGRES_HOST_PORT:-5432} -U ${POSTGRES_USER:-hillview} -d ${POSTGRES_DB:-hillview} -v ON_ERROR_STOP=1" export PGPASSWORD="${POSTGRES_PASSWORD:-hillview}" +ENRICH_API="${ENRICH_API:-http://localhost:8070}" + +# Trigger one workbench sync pass and wait it out; prints "status stats", succeeds +# only if the run did. A pass is a full reconcile: row-hash compare against the +# source, upsert new/changed rows into the mirror, stamp missing_since on rows gone +# from the source (never delete), clear it on reappearance, retire natives whose +# graduated copy is observed. The run is a background task server-side, so poll the +# status and report where it landed rather than just claiming it started. +run_sync() { + curl -sf -X POST "$ENRICH_API/api/sync/run" \ + -H 'Content-Type: application/json' -d '{}' >/dev/null || return 1 + local st="" line + for _ in $(seq 1 200); do + sleep 3 + st=$(curl -s "$ENRICH_API/api/sync/status") + case "$st" in *'"running":false'*) break ;; esac + done + line=$(echo "$st" | python3 -c 'import json,sys; r=json.load(sys.stdin)["last_runs"][0]; print(r["status"], json.dumps(r["stats"]))') || return 1 + echo "$line" + case "$line" in succeeded*) return 0 ;; *) return 1 ;; esac +} + +if [ "$MODE" = fresh ] && [ "$PRESYNC" = yes ]; then + echo "== pre-truncate mirror snapshot ==" + run_sync || { + echo "!! workbench sync failed or unreachable — refusing to truncate blind." >&2 + echo " The mirror is the only safety net for local-only rows; start the" >&2 + echo " workbench, or pass --no-presync to accept the blind spot." >&2 + exit 1 + } +fi echo "== staging ==" $PG <<'SQL' @@ -56,7 +99,9 @@ CREATE TABLE _dump_photos ( title text, keywords text, geocode text, place_name text, place_slug text, -- dump format 2 (2026-07) additions: place_parent_name text, place_parent_slug text, effective_at text, - retry_after_minutes text + retry_after_minutes text, + -- dump format 3 (2026-08) additions: + terrain_overlay text, pitch text ); CREATE TABLE _dump_anns ( id text, photo_id text, user_id text, body text, target text, @@ -112,7 +157,8 @@ INSERT INTO photos ( client_public_key_id, upload_authorized_at, processed_by_worker, processed_at, file_md5, record_created_ts, geometry, deleted, version, analysis, featured, legal_rights, title, keywords, geocode, place_name, place_slug, - place_parent_name, place_parent_slug, retry_after_minutes) + place_parent_name, place_parent_slug, retry_after_minutes, + terrain_overlay, pitch) SELECT id, NULLIF(filename,''), NULLIF(original_filename,''), NULLIF(altitude,'')::float8, NULLIF(compass_angle,'')::float8, @@ -137,7 +183,8 @@ SELECT -- BEFORE INSERT OR UPDATE trigger, COALESCE(captured_at, uploaded_at AT TIME ZONE -- 'UTC'), which overwrites anything we supply. Both inputs ARE loaded here, so the -- trigger reproduces prod's value exactly (verified: 0 rows deviate). - NULLIF(retry_after_minutes,'')::int + NULLIF(retry_after_minutes,'')::int, + NULLIF(terrain_overlay,'')::jsonb, NULLIF(pitch,'')::float8 FROM _dump_photos ON CONFLICT (id) DO NOTHING; @@ -176,17 +223,32 @@ echo "== result ==" $PG -c "SELECT 'photos' AS t, count(*) FROM photos UNION ALL SELECT 'annotations', count(*) FROM photo_annotations" echo "== mirror into the workbench ==" -if curl -sf -X POST "${ENRICH_API:-http://localhost:8070}/api/sync/run" \ - -H 'Content-Type: application/json' -d '{}' >/dev/null; then - # the sync runs in the background; report where it landed rather than just - # claiming it started - for _ in $(seq 1 200); do - sleep 3 - st=$(curl -s "${ENRICH_API:-http://localhost:8070}/api/sync/status") - case "$st" in *'"running":false'*) break ;; esac - done - echo "$st" | python3 -c 'import json,sys; r=json.load(sys.stdin)["last_runs"][0]; print(r["status"], json.dumps(r["stats"]))' +if run_sync; then + echo "== stamped-missing hygiene report ==" + # Everything the workbench reads filters missing_since IS NULL, so stamped rows + # are invisible dead weight — EXCEPT that an UNCLASSIFIED one may be real + # local-only work the mirror is the last copy of. Test-user junk is safe to + # purge by hand (in the workbench db, port ${WB_POSTGRES_HOST_PORT:-15432}). + # The users table survives reseeds, so old test-user ids still classify here. + test_ids=$($PG -Atc "SELECT COALESCE(string_agg(quote_literal(id), ','), 'NULL') + FROM users WHERE is_test OR username IN ('test','admin','testuser')") + PGPASSWORD="${WB_POSTGRES_PASSWORD:-enrich}" \ + psql -h 127.0.0.1 -p "${WB_POSTGRES_HOST_PORT:-15432}" -U "${WB_POSTGRES_USER:-enrich}" \ + -d "${WB_POSTGRES_DB:-enrich}" -v ON_ERROR_STOP=1 <&2 - echo " curl -X POST localhost:8070/api/sync/run -H 'Content-Type: application/json' -d '{}'" >&2 + echo " curl -X POST ${ENRICH_API}/api/sync/run -H 'Content-Type: application/json' -d '{}'" >&2 fi diff --git a/frontend/src/lib/components/dropdown-menu/DropdownMenu.svelte b/frontend/src/lib/components/dropdown-menu/DropdownMenu.svelte index 78f85a9e..52de475f 100644 --- a/frontend/src/lib/components/dropdown-menu/DropdownMenu.svelte +++ b/frontend/src/lib/components/dropdown-menu/DropdownMenu.svelte @@ -255,6 +255,7 @@
+ typeof raw === 'string' && raw.trim() + ? raw.split(',').map((u) => u.trim()).filter(Boolean) + : fallback; +export const ownPanoramaxInstanceUrls = parseUrlList( + import.meta.env.VITE_OWN_PANORAMAX_INSTANCE_URLS, + ['https://panoramax.hillview.cz'] +); +export const ownPhotoAssetUrlPrefixes = parseUrlList( + import.meta.env.VITE_OWN_PHOTO_ASSET_URL_PREFIXES, + ['https://pics.hillview.cz/', 'https://pics2.hillview.cz/', 'https://pics4.t3.storage.dev/'] +); + // Re-export constants for app use export { MAX_DEBUG_MODES } from './constants'; diff --git a/frontend/src/lib/sources/PanoramaxSourceLoader.test.ts b/frontend/src/lib/sources/PanoramaxSourceLoader.test.ts index aa507009..a127e99b 100644 --- a/frontend/src/lib/sources/PanoramaxSourceLoader.test.ts +++ b/frontend/src/lib/sources/PanoramaxSourceLoader.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { convertPanoramaxItem } from './PanoramaxSourceLoader'; +import { convertPanoramaxItem, isOwnInstanceItem } from './PanoramaxSourceLoader'; describe('convertPanoramaxItem', () => { const source = { id: 'panoramax', type: 'panoramax' }; @@ -145,4 +145,84 @@ describe('convertPanoramaxItem', () => { }; expect(convertPanoramaxItem(item, source)?.url).toBe('https://example.com/thumb.jpg'); }); -}); \ No newline at end of file +}); +describe('isOwnInstanceItem', () => { + const foreignItem = { + id: 'ba7c08c3-6415-4044-9894-a8c1d0e59986', + links: [ + { rel: 'via', href: 'https://panoramax.openstreetmap.fr', type: 'application/json' } + ], + assets: { + hd: { href: 'https://panoramax.openstreetmap.fr/api/pictures/x/hd.jpg' } + } + }; + + it('keeps items whose via link points at a foreign instance', () => { + expect(isOwnInstanceItem(foreignItem)).toBe(false); + }); + + it('drops items whose via link points at our instance', () => { + const item = { + ...foreignItem, + links: [{ rel: 'via', href: 'https://panoramax.hillview.cz', instance_name: 'hillview' }] + }; + expect(isOwnInstanceItem(item)).toBe(true); + }); + + it('matches via hrefs regardless of trailing slash and case', () => { + const item = { + ...foreignItem, + links: [{ rel: 'via', href: 'https://Panoramax.Hillview.CZ/' }] + }; + expect(isOwnInstanceItem(item)).toBe(true); + }); + + it('does not treat a foreign host merely containing our prefix as ours', () => { + const item = { + ...foreignItem, + links: [{ rel: 'via', href: 'https://panoramax.hillview.cz.evil.example' }] + }; + expect(isOwnInstanceItem(item)).toBe(false); + }); + + it('via link is authoritative: no asset fallback when via is present', () => { + // foreign via + our asset host: trust the via link + const item = { + ...foreignItem, + links: [{ rel: 'via', href: 'https://panoramax.openstreetmap.fr' }], + assets: { hd: { href: 'https://pics.hillview.cz/opt/2048/u/p.webp' } } + }; + expect(isOwnInstanceItem(item)).toBe(false); + }); + + it('falls back to asset-URL prefixes when no via link exists', () => { + const item = { + id: 'x', + assets: { hd: { href: 'https://pics.hillview.cz/opt/2048/u/p.webp' } } + }; + expect(isOwnInstanceItem(item)).toBe(true); + }); + + it('asset fallback recognizes every configured pics/CDN host', () => { + for (const prefix of [ + 'https://pics.hillview.cz/', + 'https://pics2.hillview.cz/', + 'https://pics4.t3.storage.dev/' + ]) { + const item = { id: 'x', assets: { thumb: { href: `${prefix}opt/640/u/p.webp` } } }; + expect(isOwnInstanceItem(item)).toBe(true); + } + }); + + it('keeps items with neither via link nor recognizable assets', () => { + const item = { + id: 'x', + assets: { hd: { href: 'https://cdn.elsewhere.example/p.jpg' } } + }; + expect(isOwnInstanceItem(item)).toBe(false); + }); + + it('tolerates items with no links and no assets', () => { + expect(isOwnInstanceItem({ id: 'x' })).toBe(false); + }); +}); diff --git a/frontend/src/lib/sources/PanoramaxSourceLoader.ts b/frontend/src/lib/sources/PanoramaxSourceLoader.ts index 250afc4c..5e6d9c09 100644 --- a/frontend/src/lib/sources/PanoramaxSourceLoader.ts +++ b/frontend/src/lib/sources/PanoramaxSourceLoader.ts @@ -11,7 +11,7 @@ import type { PhotoData, Bounds } from '../photoWorkerTypes'; import { BasePhotoSourceLoader, type PhotoSourceCallbacks } from './PhotoSourceLoader'; import type { PhotoSourceOptions } from './PhotoSourceFactory'; -import { backendUrl } from '../config'; +import { backendUrl, ownPanoramaxInstanceUrls, ownPhotoAssetUrlPrefixes } from '../config'; const LOG_PREFIX = '🢄🔍PanoramaxSourceLoader'; const doLog = false; @@ -82,6 +82,33 @@ async function ensurePanoramaxHidden( return hiddenInflight; } +const normalizeInstanceUrl = (u: string): string => u.trim().replace(/\/+$/, '').toLowerCase(); +const ownInstanceUrlsNormalized = ownPanoramaxInstanceUrls.map(normalizeInstanceUrl); + +// Self-duplicate detection: once Hillview federates, the meta-catalog serves +// our own CC photos back through this source, duplicating the native hillview +// copies. The catalog's /api/search puts a rel=via link on every item whose +// href is the origin instance URL — that's the primary signal. The asset-host +// prefix check is a fallback for responses without via links. Cross-source +// id-dedup is deliberately NOT the mechanism: the hillview source caps photos +// per bbox, so the native copy may be absent while the federated one loads. +export function isOwnInstanceItem(item: any): boolean { + const links: any[] = Array.isArray(item?.links) ? item.links : []; + const via = links.find((l) => l?.rel === 'via' && typeof l?.href === 'string'); + if (via) { + const href = normalizeInstanceUrl(via.href); + return ownInstanceUrlsNormalized.some((u) => href === u || href.startsWith(u + '/')); + } + const assets = item?.assets && typeof item.assets === 'object' ? item.assets : {}; + for (const key of Object.keys(assets)) { + const href = assets[key]?.href; + if (typeof href === 'string' && ownPhotoAssetUrlPrefixes.some((p) => href.startsWith(p))) { + return true; + } + } + return false; +} + function getProducerIdFromItem(item: any): string | undefined { const providers: any[] = Array.isArray(item?.providers) ? item.providers : []; const producer = @@ -313,6 +340,7 @@ export class PanoramaxSourceLoader extends BasePhotoSourceLoader { const items: any[] = Array.isArray(data?.features) ? data.features : []; let droppedHidden = 0; + let droppedSelf = 0; for (const item of items) { if (item?.id && hidden.photoIds.has(item.id)) { droppedHidden++; @@ -323,11 +351,15 @@ export class PanoramaxSourceLoader extends BasePhotoSourceLoader { droppedHidden++; continue; } + if (isOwnInstanceItem(item)) { + droppedSelf++; + continue; + } const photo = convertPanoramaxItem(item, this.source); if (photo) this.streamPhotos.push(photo); } - if (doLog) console.log(`${LOG_PREFIX}: got ${this.streamPhotos.length} photos for ${this.source.id} (filtered ${droppedHidden} hidden)`); + if (doLog) console.log(`${LOG_PREFIX}: got ${this.streamPhotos.length} photos for ${this.source.id} (filtered ${droppedHidden} hidden, ${droppedSelf} own-instance)`); this.callbacks.enqueueMessage({ type: 'photosAdded', diff --git a/frontend/tests-playwright/zoomview-print.spec.ts b/frontend/tests-playwright/zoomview-print.spec.ts index 9f017196..2e68578e 100644 --- a/frontend/tests-playwright/zoomview-print.spec.ts +++ b/frontend/tests-playwright/zoomview-print.spec.ts @@ -90,9 +90,10 @@ test.describe('Zoom view print view', () => { // back at the visitor's value await page.click('[data-testid="osd-display-menu-toggle"]', { timeout: T(10000) }); await expect(page.locator('[data-testid="osd-annotation-scale-slider"]')).toHaveValue('1'); - // toggle it closed with its own button — Escape on an open menu falls + // close it the way a visitor does, on the backdrop — the toggle is under + // that backdrop while the menu is open, and Escape on an open menu falls // through to closing the viewer (the dropdown eats the key first) - await page.click('[data-testid="osd-display-menu-toggle"]', { timeout: T(10000) }); + await page.click('[data-testid="dropdown-backdrop"]', { position: { x: 5, y: 5 }, timeout: T(10000) }); await expect(page.locator('[data-testid="osd-annotation-scale-slider"]')).not.toBeVisible(); // the Exit button does the same diff --git a/scripts/check_container_freshness.py b/scripts/check_container_freshness.py index 5247e8b1..7ec74683 100755 --- a/scripts/check_container_freshness.py +++ b/scripts/check_container_freshness.py @@ -50,16 +50,31 @@ TARGETS = {3000: FRONTEND_TARGET} -# Hosts whose origin is a Caddy vhost proxying into the built frontend container +# Hosts whose origin is a Caddy vhost fronting the built frontend container # rather than the dev server. Keyed by host because they answer on :443, which no # port rule can distinguish from any other https origin — and these are exactly -# the profiles that serve compiled output, so skipping them would disable this -# check precisely where it is needed. Keep in sync with scripts/host_profiles.py. -CADDY_FRONTEND_HOSTS = { - "hv.jj.internal", - "hv.dev4.local", - "hv.dev4.jj.internal", -} +# the profiles that serve compiled output, so missing one disables this check +# precisely where it is needed. +# +# DERIVED from the profiles, not listed: as a hand-written set it had already +# fallen behind, naming only jj and dev4 while dev4-2 and dev4-3 existed. +def _caddy_frontend_hosts(): + try: + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from host_profiles import PROFILES + except ImportError: + return set() + hosts = set() + for profile in PROFILES.values(): + backend = profile.get("frontend_env", {}).get("VITE_BACKEND", "") or "" + if backend.startswith("https://"): # https => behind a Caddy vhost + host = urllib.parse.urlparse(backend).hostname + if host: + hosts.add(host) + return hosts + + +CADDY_FRONTEND_HOSTS = _caddy_frontend_hosts() MAX_LISTED = 12 @@ -245,7 +260,12 @@ def main(): target, _url = resolve_target() if target is None: - return 0 + # No Playwright target here — but "was this image built for the config it + # is running under?" is true or false regardless of whether anything + # tests it, and the interactive-only profiles (frontend_url: None) are + # exactly where a stale bundle goes unnoticed. So still check the bundle; + # only the source-staleness half depends on knowing the target. + return 1 if (check_baked_origin(FRONTEND_TARGET["container"]) and opts.strict) else 0 container = target["container"] built_at = image_built_at(container) diff --git a/scripts/host_profiles.py b/scripts/host_profiles.py index 5721a2ba..54bef71b 100644 --- a/scripts/host_profiles.py +++ b/scripts/host_profiles.py @@ -194,8 +194,8 @@ def caddy(host, frontend_url, **extra): # to be turned on, or it reverted every boot), and its own names resolve to # itself rather than to dev4. **vm_profiles("dev4", "192.168.122.64"), - **vm_profiles("dev4-3", "192.168.122.31", ygg_reachable_locally=True), - **vm_profiles("dev4-2", "192.168.122.37", ygg_reachable_locally=True), + **vm_profiles("dev4-3", "192.168.122.31"), + **vm_profiles("dev4-2", "192.168.122.37"), } @@ -211,7 +211,14 @@ def caddy(host, frontend_url, **extra): # # Driving a browser wants an h2 origin either way — plain HTTP/1.1 starves chunk # loads into the stalled-request flake class. -WEB_SUFFIXES = ("local", "ygg") +# +# ygg first: it is the canonical origin, the one name that works from anywhere. +# `local` is the fallback for a box that cannot reach its own ygg name, which is +# every VM — their ygg names are terminated on jj, and ygg does not run inside +# them. The reachable_locally flag is what makes this ordering safe, and it means +# a box that later CAN resolve its own ygg name starts preferring it with no +# change here. +WEB_SUFFIXES = ("ygg", "local") def profile_for(hostname, leg):