diff --git a/alembic/versions/0019_allowlist_sealed_manifest_hashes.py b/alembic/versions/0019_allowlist_sealed_manifest_hashes.py new file mode 100644 index 00000000..a21f1d54 --- /dev/null +++ b/alembic/versions/0019_allowlist_sealed_manifest_hashes.py @@ -0,0 +1,46 @@ +"""Add sealed_manifest_hashes to image_digest_allowlist. + +Stores the build-time sealed surface (path → 64-hex SHA-256) alongside each +BASE-produced digest so production can supply prism's non-empty +``expected_sealed_manifest_hashes``. Application registration rejects empty +maps; the column uses a non-null JSON server default of ``{}`` so existing +rows migrate without rewrite. Legacy empty maps cannot be re-registered and +fail closed if loaded into DigestRecord until backfilled. + +Revision ID: 0019_allowlist_sealed_hashes +Revises: 0018_attestation_nonces +Create Date: 2026-07-27 00:00:00.000000 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "0019_allowlist_sealed_hashes" +down_revision: str | None = "0018_attestation_nonces" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Apply the migration.""" + + op.add_column( + "image_digest_allowlist", + sa.Column( + "sealed_manifest_hashes", + sa.JSON(), + server_default=sa.text("'{}'"), + nullable=False, + ), + ) + + +def downgrade() -> None: + """Revert the migration.""" + + op.drop_column("image_digest_allowlist", "sealed_manifest_hashes") diff --git a/packages/challenges/prism/src/prism_challenge/attestation_routes.py b/packages/challenges/prism/src/prism_challenge/attestation_routes.py index 9a09804b..e26c85c1 100644 --- a/packages/challenges/prism/src/prism_challenge/attestation_routes.py +++ b/packages/challenges/prism/src/prism_challenge/attestation_routes.py @@ -61,6 +61,7 @@ class RegisterDigestBody(BaseModel): tree_sha: str variant: str digest: str + sealed_manifest_hashes: dict[str, str] class CheckAllowlistBody(BaseModel): @@ -322,12 +323,19 @@ def build_attestation_internal_router() -> APIRouter: async def register_digest(request: Request, body: RegisterDigestBody) -> dict[str, str]: ensure_default_constation_services(request.app) allowlist: DigestAllowlist = request.app.state.digest_allowlist - record = DigestRecord( - commit_sha=body.commit_sha, - tree_sha=body.tree_sha, - variant=ImageVariant(body.variant.strip().lower()), - digest=body.digest, - ) + try: + record = DigestRecord( + commit_sha=body.commit_sha, + tree_sha=body.tree_sha, + variant=ImageVariant(body.variant.strip().lower()), + digest=body.digest, + sealed_manifest_hashes=body.sealed_manifest_hashes, + ) + except ValueError as exc: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=str(exc), + ) from exc allowlist.register(record) return {"status": "registered", "digest": record.digest} diff --git a/scripts/e2e_lium_attestation.py b/scripts/e2e_lium_attestation.py index 43c62c9c..c0b15309 100644 --- a/scripts/e2e_lium_attestation.py +++ b/scripts/e2e_lium_attestation.py @@ -208,7 +208,15 @@ async def get_pod_raw(self, pod_id: str) -> LiumPodRead: pod_id=pod_id, template_id="tmpl-e2e-1", docker_image_digest=digest, - raw={"id": pod_id, "template": {"docker_image_digest": digest}}, + raw={ + "id": pod_id, + "status": "RUNNING", + "executor": { + "miner_hotkey": HOTKEY, + "executor_ip_address": "10.0.0.1", + }, + "template": {"docker_image_digest": digest}, + }, ) async def balance(self) -> float: @@ -374,6 +382,7 @@ async def _probe(client: Any) -> None: work_unit_id="wu-placeholder", # overwritten after submission seed pod_id=POD_ID, duration_seconds=10.0 if not adversarial else 10.0, + required_digest=DIGEST_HONEST, ) ) @@ -386,14 +395,14 @@ async def run_offline(mode: Mode) -> dict[str, Any]: # --- 1. Allowlist: BASE-produced digest registration (submission identity) --- allowlist = DigestAllowlist() - allowlist.register( - DigestRecord( - commit_sha=COMMIT_SHA, - tree_sha=TREE_SHA, - variant=VARIANT, - digest=DIGEST_HONEST, - ) + honest_record = DigestRecord( + commit_sha=COMMIT_SHA, + tree_sha=TREE_SHA, + variant=VARIANT, + digest=DIGEST_HONEST, + sealed_manifest_hashes=dict(SEALED_MANIFEST), ) + allowlist.register(honest_record) # --- 2. Continuous constation runner/poller (fixture Lium, real runner) --- # First pass uses placeholder work_unit; we re-bind nonce to real submission id. @@ -467,7 +476,7 @@ def verify_signature(_signed: object) -> Any: pod_id=POD_ID, nonce=issued.nonce, signed_attestation={"sig": "fixture-ok"}, - expected_sealed_manifest_hashes=dict(SEALED_MANIFEST), + expected_sealed_manifest_hashes=dict(honest_record.sealed_manifest_hashes), reported_sealed_manifest_hashes=dict(SEALED_MANIFEST), lium_declared_digest=lium_for_bundle, constation_gap_budget_seconds=gap_budget, @@ -584,12 +593,12 @@ def _assert_honest(bag: dict[str, Any]) -> list[str]: def _assert_adversarial(bag: dict[str, Any]) -> list[str]: errors: list[str] = [] - # Runner must fail closed on mid-run swap (corroboration mismatch). + # Runner must fail closed on mid-run swap (triangle: sidecar != required). if bag["run_record_ok"]: errors.append("runner expected fail on mid-run digest swap") - if bag["run_record_reason"] != ConstationFailCode.CORROBORATION_MISMATCH.value: + if bag["run_record_reason"] != ConstationFailCode.REQUIRED_DIGEST_MISMATCH.value: errors.append( - f"runner reason expected corroboration_mismatch got {bag['run_record_reason']}" # noqa: E501 + f"runner reason expected required_digest_mismatch got {bag['run_record_reason']}" # noqa: E501 ) if bag["run_record_fault_class"] != FaultClass.MINER.value: errors.append( diff --git a/scripts/prism_lium_attestation_e2e.py b/scripts/prism_lium_attestation_e2e.py index 9a6bddac..9bba32be 100644 --- a/scripts/prism_lium_attestation_e2e.py +++ b/scripts/prism_lium_attestation_e2e.py @@ -183,16 +183,27 @@ async def attest(self, *, pod_id: str, phase: str) -> str: @dataclass class ScriptedLium: digests: list[str | None] + miner_hotkey: str = HOTKEY + status: str = "RUNNING" calls: int = 0 async def get_pod_raw(self, pod_id: str) -> LiumPodRead: self.calls += 1 idx = min(self.calls - 1, len(self.digests) - 1) + digest = self.digests[idx] return LiumPodRead( pod_id=pod_id, template_id="tmpl-fixture", - docker_image_digest=self.digests[idx], - raw={}, + docker_image_digest=digest, + raw={ + "id": pod_id, + "status": self.status, + "executor": { + "miner_hotkey": self.miner_hotkey, + "executor_ip_address": "10.0.0.1", + }, + "template": {"docker_image_digest": digest}, + }, ) async def balance(self) -> float: @@ -264,6 +275,7 @@ def _register_allowlist(allowlist: DigestAllowlist, digest: str) -> None: tree_sha=TREE, variant=ImageVariant.CUDA, digest=digest, + sealed_manifest_hashes=dict(SEALED), ) ) @@ -447,7 +459,7 @@ async def sleep_fn(seconds: float) -> None: work_unit_id=work_unit_id, pod_id=pod_id, duration_seconds=duration, - expected_sidecar_digest=sidecar.digest, + required_digest=sidecar.digest, ) ) diff --git a/scripts/surface_constation.sh b/scripts/surface_constation.sh new file mode 100755 index 00000000..900a3cf4 --- /dev/null +++ b/scripts/surface_constation.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# T12: production constation surface scenarios S1–S8 + B1s/B2s. +# Exit non-zero on any failure. No live Lium. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +export UV_CACHE_DIR="${UV_CACHE_DIR:-/var/tmp/uv-cache}" +# Prism optional surface pieces (S3/S6) + local surface helpers. +export PYTHONPATH="${ROOT}/packages/challenges/prism/src:${ROOT}/tests/surface${PYTHONPATH:+:${PYTHONPATH}}" + +SURFACE_NODEIDS=( + "tests/surface/test_constation_production_surface.py::test_s1_honest_path_seal_store_forward_nests_bundle" + "tests/surface/test_constation_production_surface.py::test_s2_adversarial_triangle_fail_no_ok_bundle_put" + "tests/surface/test_constation_production_surface_prism_http.py::test_s3_missing_bundle_prism_behavior" + "tests/surface/test_constation_production_surface.py::test_s4_nonce_issue_seal_consume_once_ok" + "tests/surface/test_constation_production_surface.py::test_s5_allowlist_sealed_hashes_required" + "tests/surface/test_constation_production_surface_prism_http.py::test_s6_legacy_gate_no_elevation_without_constation" + "tests/surface/test_constation_production_surface.py::test_s7_forwarder_embed_from_lookup" + "tests/surface/test_constation_production_surface_prism_http.py::test_s8_challenge_route_smoke" + "tests/surface/test_constation_production_surface.py::test_b1s_verify_key_wiring_smoke" + "tests/surface/test_constation_production_surface.py::test_b2s_orchestrator_source_never_calls_consume" +) + +echo "==> surface_constation: UV_CACHE_DIR=${UV_CACHE_DIR}" +echo "==> running ${#SURFACE_NODEIDS[@]} node ids" + +exec uv run pytest -q --tb=short "${SURFACE_NODEIDS[@]}" diff --git a/src/base/cli_app/main.py b/src/base/cli_app/main.py index 6b51c5f8..d4c7bdd3 100644 --- a/src/base/cli_app/main.py +++ b/src/base/cli_app/main.py @@ -550,6 +550,7 @@ def _master_orchestration_driver( worker_service: WorkerCoordinationService | None = None, worker_assignment_service: WorkerAssignmentService | None = None, bundle_store: Any | None = None, + constation_hook: Any | None = None, ) -> MasterOrchestrationDriver: """Build the live master orchestration driver (architecture.md sec 4). @@ -602,6 +603,7 @@ def _bundle_lookup(work_unit_id: str): retries=settings.master.challenge_retries, bundle_lookup=_bundle_lookup if bundle_store is not None else None, ), + constation_hook=constation_hook, ) return MasterOrchestrationDriver( assignment_service=assignment_service, @@ -1147,7 +1149,12 @@ def master_proxy(config: Path = typer.Option(Path("config/master.example.yaml")) from datetime import timedelta from base.master.constation.allowlist_repository import DigestAllowlistRepository + from base.master.constation.attestation_keys import load_attestation_verify_key from base.master.constation.bundle_store import ConstationBundleStore + from base.master.constation.custody_keys import ( + build_constation_runtime, + make_constation_pre_forward_hook, + ) from base.master.constation.nonce_repository import DurableAttestationNonceService from base.master.constation.routes import build_constation_router @@ -1156,6 +1163,19 @@ def master_proxy(config: Path = typer.Option(Path("config/master.example.yaml")) constation_nonce_service = DurableAttestationNonceService( session_factory, ttl=timedelta(hours=2) ) + # Custody + MinerPodBinding + ProductionConstationOrchestrator when enabled + # and custody master key is present (fail-closed otherwise; master still boots). + constation_runtime = build_constation_runtime( + settings, + nonce_service=constation_nonce_service, + bundle_store=constation_bundle_store, + ) + constation_pod_binding = constation_runtime.pod_binding + constation_orchestrator = constation_runtime.orchestrator + constation_hook = make_constation_pre_forward_hook( + constation_orchestrator, + duration_seconds=float(settings.constation.duration_seconds), + ) # Internal token for master constation check_*/issue/register/bundle. # Reuse admin token when present so ops and prism can share one secret. constation_internal_token = ( @@ -1170,6 +1190,8 @@ def master_proxy(config: Path = typer.Option(Path("config/master.example.yaml")) nonce_service=constation_nonce_service, bundle_store=constation_bundle_store, internal_token=str(constation_internal_token), + attestation_verify_key=load_attestation_verify_key(settings), + pod_binding=constation_pod_binding, ) orchestration_driver = _master_orchestration_driver( @@ -1180,6 +1202,7 @@ def master_proxy(config: Path = typer.Option(Path("config/master.example.yaml")) worker_service=worker_service, worker_assignment_service=worker_assignment_service, bundle_store=constation_bundle_store, + constation_hook=constation_hook, ) # Registry-driven challenge deploy (architecture.md sec 4 + sec 9.2): the # master reconcile loop turns every ACTIVE registry challenge into a running @@ -1256,6 +1279,11 @@ def master_proxy(config: Path = typer.Option(Path("config/master.example.yaml")) ), constation_router=constation_router, ) + # Expose constation services for worker-path / ops reachability (optional). + if constation_pod_binding is not None: + proxy.state.constation_pod_binding = constation_pod_binding + if constation_orchestrator is not None: + proxy.state.constation_orchestrator = constation_orchestrator endpoint = f"{settings.master.proxy_host}:{settings.master.proxy_port}" typer.echo(f"Starting proxy API on {endpoint}") uvicorn.run(proxy, host=settings.master.proxy_host, port=settings.master.proxy_port) diff --git a/src/base/compute/__init__.py b/src/base/compute/__init__.py index 5854a8c8..e17d3cd1 100644 --- a/src/base/compute/__init__.py +++ b/src/base/compute/__init__.py @@ -24,6 +24,11 @@ LiumKeyCustody, generate_custody_master_key, ) +from base.compute.constation_pod import ( + assert_pod_bound, + extract_miner_hotkey, + pod_is_running, +) from base.compute.constation_poller import ( ContinuousConstationPoller, PollerConfig, @@ -33,6 +38,22 @@ ConstationRunner, ConstationRunRequest, ) +from base.compute.constation_sidecar_client import ( + HttpSidecarAttestor, + SidecarAttestHit, + SidecarAttestMiss, + SidecarAttestResult, +) +from base.compute.constation_sidecar_endpoint import ( + SidecarEndpointHit, + SidecarEndpointMiss, + SidecarEndpointMissReason, + resolve_sidecar_base_url, +) +from base.compute.constation_triangle import ( + DigestTriangleResult, + evaluate_digest_triangle, +) from base.compute.constation_types import ( ConstationFailCode, ConstationRunRecord, @@ -146,5 +167,18 @@ "PollerConfig", "PollerResult", "evaluate_corroboration", + "DigestTriangleResult", + "evaluate_digest_triangle", "generate_custody_master_key", + "HttpSidecarAttestor", + "SidecarAttestHit", + "SidecarAttestMiss", + "SidecarAttestResult", + "SidecarEndpointHit", + "SidecarEndpointMiss", + "SidecarEndpointMissReason", + "resolve_sidecar_base_url", + "assert_pod_bound", + "extract_miner_hotkey", + "pod_is_running", ] diff --git a/src/base/compute/constation_custody.py b/src/base/compute/constation_custody.py index 29898e2d..a1f1b358 100644 --- a/src/base/compute/constation_custody.py +++ b/src/base/compute/constation_custody.py @@ -119,6 +119,18 @@ async def register(self, *, miner_hotkey: str, api_key: str) -> ConstationVerdic logger.info("lium key registered (encrypted) for miner_hotkey=%s", hotkey) return ConstationVerdict(ok=True, reason=ConstationFailCode.OK) + def store_probed_key(self, *, miner_hotkey: str, api_key: str) -> None: + """Encrypt-at-rest after the caller has already validated the key. + + No network I/O. Used by pod binding after probe + assert_pod_bound so a + failed bind never leaves a half-registered key. + """ + hotkey = _require_nonblank("miner_hotkey", miner_hotkey) + key = _require_nonblank("api_key", api_key) + token = self._fernet.encrypt(key.encode("utf-8")) + self._by_hotkey[hotkey] = token + logger.info("lium key stored (encrypted) for miner_hotkey=%s", hotkey) + def unlock_api_key(self, miner_hotkey: str) -> str: """Decrypt the stored key for in-process use. Raises if missing/corrupt.""" hotkey = _require_nonblank("miner_hotkey", miner_hotkey) diff --git a/src/base/compute/constation_pod.py b/src/base/compute/constation_pod.py new file mode 100644 index 00000000..604677bb --- /dev/null +++ b/src/base/compute/constation_pod.py @@ -0,0 +1,70 @@ +"""Pure helpers: miner hotkey + running status from a raw Lium pod payload. + +Fail-closed. Reads only documented Lium ``PodDetailResponse`` fields: + +* ``executor.miner_hotkey`` — never invents or falls back to top-level keys +* top-level ``status`` — running iff case-insensitive equality with ``RUNNING`` + +No network I/O. Callers pass ``LiumPodRead.raw`` (or any equivalent mapping). +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from base.compute.constation_types import ConstationFailCode, ConstationVerdict + +_RUNNING_STATUS = "RUNNING" + + +def extract_miner_hotkey(pod_raw: Mapping[str, Any]) -> str | None: + """Return stripped ``executor.miner_hotkey``, or None if absent/unusable.""" + executor = pod_raw.get("executor") + if not isinstance(executor, Mapping): + return None + raw = executor.get("miner_hotkey") + if not isinstance(raw, str): + return None + hotkey = raw.strip() + if not hotkey: + return None + return hotkey + + +def pod_is_running(pod_raw: Mapping[str, Any]) -> bool: + """True only when top-level ``status`` is case-insensitively ``RUNNING``.""" + status = pod_raw.get("status") + if not isinstance(status, str): + return False + return status.strip().upper() == _RUNNING_STATUS + + +def assert_pod_bound( + *, + pod_raw: Mapping[str, Any], + expected_hotkey: str, +) -> ConstationVerdict: + """Fail-closed bind check: hotkey match then running status. + + Precedence: + 1. missing/blank/mismatched ``executor.miner_hotkey`` → POD_HOTKEY_MISMATCH + 2. status not RUNNING → POD_NOT_RUNNING + 3. otherwise ok + """ + want = expected_hotkey.strip() + got = extract_miner_hotkey(pod_raw) + if got is None or got != want: + return ConstationVerdict( + ok=False, reason=ConstationFailCode.POD_HOTKEY_MISMATCH + ) + if not pod_is_running(pod_raw): + return ConstationVerdict(ok=False, reason=ConstationFailCode.POD_NOT_RUNNING) + return ConstationVerdict(ok=True, reason=ConstationFailCode.OK) + + +__all__ = [ + "assert_pod_bound", + "extract_miner_hotkey", + "pod_is_running", +] diff --git a/src/base/compute/constation_runner.py b/src/base/compute/constation_runner.py index 7bb321b6..517c65d8 100644 --- a/src/base/compute/constation_runner.py +++ b/src/base/compute/constation_runner.py @@ -1,7 +1,9 @@ """Constation runner — master/worker-plane service entry (todos 15–17). Orchestrates encrypted key custody, continuous polling, Lium declared-digest -reads, sidecar attestation, and same-account **corroboration** (negative only). +reads, sidecar attestation, pod bind checks, and three-way digest triangle +(required / Lium-declared / sidecar-actual). Same-account corroboration is +subsumed by the triangle (negative only). Callers ------- @@ -11,19 +13,32 @@ Outputs a :class:`~base.compute.constation_types.ConstationRunRecord` whose fields map onto prism ``ConstationBundle`` gap / digest / corroboration inputs. -Agreement on corroboration never elevates by itself. +Agreement on the triangle never elevates by itself. """ from __future__ import annotations import logging -from collections.abc import Awaitable, Callable -from dataclasses import dataclass -from typing import Protocol +import secrets +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, field +from typing import Any, Protocol, cast -from base.compute.constation_corroboration import evaluate_corroboration from base.compute.constation_custody import LiumKeyCustody +from base.compute.constation_pod import assert_pod_bound from base.compute.constation_poller import ContinuousConstationPoller, PollerConfig +from base.compute.constation_sidecar_client import ( + DEFAULT_TIMEOUT_SECONDS, + HttpSidecarAttestor, + SidecarAttestHit, + SidecarAttestMiss, +) +from base.compute.constation_sidecar_endpoint import ( + SidecarEndpointMiss, + SidecarEndpointMissReason, + resolve_sidecar_base_url, +) +from base.compute.constation_triangle import evaluate_digest_triangle from base.compute.constation_types import ( ConstationFailCode, ConstationRunRecord, @@ -33,7 +48,13 @@ PollSample, fault_class_for, ) -from base.compute.lium import LiumAuthError, LiumClient, LiumError, LiumRateLimitError +from base.compute.lium import ( + LiumAuthError, + LiumClient, + LiumError, + LiumPodRead, + LiumRateLimitError, +) logger = logging.getLogger(__name__) @@ -54,17 +75,26 @@ class ConstationRunRequest: work_unit_id: str pod_id: str duration_seconds: float - expected_sidecar_digest: str | None = None + required_digest: str + # Back-compat alias accepted by older call sites via keyword only if needed. + # Prefer ``required_digest`` (allowlist / BASE required image digest). NowFn = Callable[[], float] SleepFn = Callable[[float], Awaitable[None]] RngFn = Callable[[], float] +AttestorFactory = Callable[[Mapping[str, Any]], object] +NonceFn = Callable[[], str] + + +def _default_poll_nonce() -> str: + """Ephemeral poll nonce — not durable-consumed (seal nonce is separate).""" + return secrets.token_hex(16) @dataclass class ConstationRunner: - """Service entry: custody + poller + corroboration → run record.""" + """Service entry: custody + poller + triangle → run record.""" custody: LiumKeyCustody sidecar: SidecarAttestor @@ -72,9 +102,15 @@ class ConstationRunner: now_fn: NowFn sleep_fn: SleepFn rng_fn: RngFn + attestor_factory: AttestorFactory | None = None + sidecar_internal_port: int | None = None + sidecar_timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS + poll_nonce_fn: NonceFn | None = None + last_signed_wire: Mapping[str, Any] | None = field(default=None, init=False) async def run(self, request: ConstationRunRequest) -> ConstationRunRecord: """Execute continuous constation for ``request``; fail closed on faults.""" + self.last_signed_wire = None hotkey = request.miner_hotkey.strip() if not self.custody.has_key(hotkey): return _record( @@ -125,9 +161,7 @@ async def poll_once(phase: str) -> PollSample | ConstationVerdict: last_lium = ( result.samples[-1].lium_declared_digest if result.samples else None ) - status = CorroborationStatus.NOT_EVALUATED - if result.reason is ConstationFailCode.CORROBORATION_MISMATCH: - status = CorroborationStatus.MISMATCH + status = _status_for_fail(result.reason) return _record( request, ok=False, @@ -155,40 +189,37 @@ async def poll_once(phase: str) -> PollSample | ConstationVerdict: samples=(), ) - # Final corroboration across last sample (and ensure no sample mismatched) + # Final triangle across every sample (fail-closed; triangle > two-way). for sample in result.samples: - corr = evaluate_corroboration( - lium_declared_digest=sample.lium_declared_digest, - sidecar_digest=sample.sidecar_digest, + tri = evaluate_digest_triangle( + required=request.required_digest, + lium_declared=sample.lium_declared_digest, + sidecar=sample.sidecar_digest, ) - if not corr.ok: + if not tri.ok: + fail = tri.fail_code or ConstationFailCode.CORROBORATION_MISMATCH return _record( request, ok=False, - reason=ConstationFailCode.CORROBORATION_MISMATCH, + reason=fail, gap_budget=result.gap_budget_seconds, observed_gap=result.observed_max_gap_seconds, - sidecar=corr.sidecar_digest, - lium=corr.lium_declared_digest, - status=CorroborationStatus.MISMATCH, + sidecar=sample.sidecar_digest, + lium=sample.lium_declared_digest, + status=_status_for_fail(fail), samples=result.samples, - detail=corr.verdict.detail, ) last = result.samples[-1] - corr = evaluate_corroboration( - lium_declared_digest=last.lium_declared_digest, - sidecar_digest=last.sidecar_digest, - ) return _record( request, ok=True, reason=ConstationFailCode.OK, gap_budget=result.gap_budget_seconds, observed_gap=result.observed_max_gap_seconds, - sidecar=corr.sidecar_digest, - lium=corr.lium_declared_digest, - status=corr.status, + sidecar=last.sidecar_digest, + lium=last.lium_declared_digest, + status=CorroborationStatus.AGREE, samples=result.samples, ) @@ -216,14 +247,127 @@ async def _poll_once( # Re-raise so poller applies bounded network retry raise + bound = assert_pod_bound( + pod_raw=pod.raw, + expected_hotkey=request.miner_hotkey, + ) + if not bound.ok: + return ConstationVerdict( + ok=False, + reason=bound.reason, + detail=f"phase={phase}", + fault_class=bound.fault_class, + ) + + obtained = await self._obtain_sidecar_digest( + pod=pod, request=request, phase=phase + ) + if isinstance(obtained, ConstationVerdict): + return obtained + sidecar_digest, wire = obtained + if wire is not None: + self.last_signed_wire = wire + + tri = evaluate_digest_triangle( + required=request.required_digest, + lium_declared=pod.docker_image_digest, + sidecar=sidecar_digest, + ) + if not tri.ok: + fail = tri.fail_code or ConstationFailCode.CORROBORATION_MISMATCH + return ConstationVerdict( + ok=False, + reason=fail, + detail=f"phase={phase}", + ) + + return PollSample( + at_monotonic=self.now_fn(), + phase=phase, + sidecar_digest=sidecar_digest, + lium_declared_digest=pod.docker_image_digest, + ) + + async def _obtain_sidecar_digest( + self, + *, + pod: LiumPodRead, + request: ConstationRunRequest, + phase: str, + ) -> tuple[str, Mapping[str, Any] | None] | ConstationVerdict: + """Resolve attestor (factory / HTTP dial / injected) and fetch digest.""" + attestor: object + if self.attestor_factory is not None: + try: + attestor = self.attestor_factory(pod.raw) + except Exception as exc: + logger.warning( + "attestor_factory failed pod=%s phase=%s err=%s", + request.pod_id, + phase, + type(exc).__name__, + ) + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.SIDECAR_ATTEST_FAILED, + detail=f"phase={phase} err={type(exc).__name__}", + ) + elif self.sidecar_internal_port is not None: + resolved = resolve_sidecar_base_url( + pod.raw, + internal_port=self.sidecar_internal_port, + ) + if isinstance(resolved, SidecarEndpointMiss): + return ConstationVerdict( + ok=False, + reason=_endpoint_miss_to_fail(resolved.reason), + detail=f"phase={phase} endpoint={resolved.reason.value}", + ) + attestor = HttpSidecarAttestor( + base_url=resolved.base_url, + timeout_seconds=self.sidecar_timeout_seconds, + ) + else: + attestor = self.sidecar + + return await self._call_attestor( + attestor, + pod_id=request.pod_id, + phase=phase, + ) + + async def _call_attestor( + self, + attestor: object, + *, + pod_id: str, + phase: str, + ) -> tuple[str, Mapping[str, Any] | None] | ConstationVerdict: try: - sidecar_digest = await self.sidecar.attest( - pod_id=request.pod_id, phase=phase + if isinstance(attestor, HttpSidecarAttestor): + nonce_fn = self.poll_nonce_fn or _default_poll_nonce + result = await attestor.attest(nonce=nonce_fn(), phase=phase) + if isinstance(result, SidecarAttestMiss): + return ConstationVerdict( + ok=False, + reason=result.reason, + detail=result.detail or f"phase={phase}", + ) + if isinstance(result, SidecarAttestHit): + return result.digest, result.wire + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.SIDECAR_RESPONSE_INVALID, + detail=f"phase={phase}", + ) + + digest = await cast(SidecarAttestor, attestor).attest( + pod_id=pod_id, phase=phase ) except Exception as exc: logger.warning( "sidecar attest failed pod=%s phase=%s err=%s", - request.pod_id, + pod_id, phase, type(exc).__name__, ) @@ -233,24 +377,25 @@ async def _poll_once( detail=f"phase={phase} err={type(exc).__name__}", ) - lium_digest = pod.docker_image_digest - corr = evaluate_corroboration( - lium_declared_digest=lium_digest, - sidecar_digest=sidecar_digest, - ) - if not corr.ok: + if not isinstance(digest, str) or not digest.strip(): return ConstationVerdict( ok=False, - reason=ConstationFailCode.CORROBORATION_MISMATCH, - detail=corr.verdict.detail, + reason=ConstationFailCode.SIDECAR_RESPONSE_INVALID, + detail=f"phase={phase}", ) + return digest, None - return PollSample( - at_monotonic=self.now_fn(), - phase=phase, - sidecar_digest=corr.sidecar_digest, - lium_declared_digest=corr.lium_declared_digest, - ) + +def _endpoint_miss_to_fail(reason: SidecarEndpointMissReason) -> ConstationFailCode: + if reason is SidecarEndpointMissReason.SIDECAR_PORT_UNPUBLISHED: + return ConstationFailCode.SIDECAR_PORT_UNPUBLISHED + return ConstationFailCode.SIDECAR_ATTEST_FAILED + + +def _status_for_fail(reason: ConstationFailCode) -> CorroborationStatus: + if reason is ConstationFailCode.CORROBORATION_MISMATCH: + return CorroborationStatus.MISMATCH + return CorroborationStatus.NOT_EVALUATED def _record( @@ -285,6 +430,7 @@ def _record( __all__ = [ + "AttestorFactory", "ConstationRunRequest", "ConstationRunner", "SidecarAttestor", diff --git a/src/base/compute/constation_sidecar_client.py b/src/base/compute/constation_sidecar_client.py new file mode 100644 index 00000000..e9c1268b --- /dev/null +++ b/src/base/compute/constation_sidecar_client.py @@ -0,0 +1,285 @@ +"""HTTP SidecarAttestor — BASE dials recipe listen ``POST /v1/sidecar/attest``. + +Transport only: POST nonce+phase, parse signed wire, extract digest. Does not +consume nonces or verify HMAC (callers use ``base.attestation.payload``). +Fail-closed to typed ``ConstationFailCode`` outcomes for the runner. +""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping +from dataclasses import dataclass +from enum import StrEnum +from typing import Any, Final + +import httpx + +from base.compute.constation_types import ConstationFailCode + +logger = logging.getLogger(__name__) + +ATTEST_PATH: Final[str] = "/v1/sidecar/attest" +DEFAULT_TIMEOUT_SECONDS: Final[float] = 5.0 +_VALID_PHASES: Final[frozenset[str]] = frozenset({"start", "interval", "end"}) + + +class SidecarAttestPhase(StrEnum): + """Phases accepted by recipe listen mode.""" + + START = "start" + INTERVAL = "interval" + END = "end" + + +@dataclass(frozen=True, slots=True) +class SidecarAttestHit: + """Successful transport parse of a sidecar attestation wire body.""" + + digest: str + nonce: str + pod_id: str + phase: str + signature: str + algorithm: str + schema_version: str + sealed_manifest_hashes: Mapping[str, str] + wire: Mapping[str, Any] + + def __bool__(self) -> bool: + return True + + +@dataclass(frozen=True, slots=True) +class SidecarAttestMiss: + """Fail-closed transport or parse outcome.""" + + reason: ConstationFailCode + detail: str | None = None + + def __bool__(self) -> bool: + return False + + +SidecarAttestResult = SidecarAttestHit | SidecarAttestMiss + + +@dataclass(frozen=True, slots=True) +class HttpSidecarAttestor: + """Async HTTP client for recipe sidecar listen mode (dial-out). + + ``base_url`` is the resolved origin (see ``resolve_sidecar_base_url``). + ``timeout_seconds`` bounds connect+read (must be positive). + ``transport`` is optional for tests (respx / MockTransport). + """ + + base_url: str + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS + transport: httpx.AsyncBaseTransport | None = None + + def __post_init__(self) -> None: + url = self.base_url.strip().rstrip("/") + if not url: + raise ValueError("base_url must be a non-empty HTTP(S) origin") + if self.timeout_seconds <= 0: + raise ValueError( + f"timeout_seconds must be positive, got {self.timeout_seconds!r}" + ) + object.__setattr__(self, "base_url", url) + + async def attest(self, *, nonce: str, phase: str) -> SidecarAttestResult: + """POST attest path; extract payload digest. Does not consume nonce.""" + try: + nonce_s = _require_nonblank("nonce", nonce) + phase_s = _normalize_phase(phase) + except ValueError as exc: + return SidecarAttestMiss( + reason=ConstationFailCode.SIDECAR_RESPONSE_INVALID, + detail=str(exc), + ) + + url = f"{self.base_url}{ATTEST_PATH}" + body = {"nonce": nonce_s, "phase": phase_s} + try: + async with httpx.AsyncClient( + timeout=httpx.Timeout(self.timeout_seconds), + transport=self.transport, + ) as client: + response = await client.post(url, json=body) + except httpx.TimeoutException as exc: + logger.warning( + "sidecar attest timeout url=%s phase=%s err=%s", + url, + phase_s, + type(exc).__name__, + ) + return SidecarAttestMiss( + reason=ConstationFailCode.NETWORK_PARTITION, + detail=f"timeout phase={phase_s}", + ) + except httpx.HTTPError as exc: + logger.warning( + "sidecar attest transport failed url=%s phase=%s err=%s", + url, + phase_s, + type(exc).__name__, + ) + return SidecarAttestMiss( + reason=ConstationFailCode.SIDECAR_ATTEST_FAILED, + detail=f"transport phase={phase_s} err={type(exc).__name__}", + ) + + return _parse_response(response, requested_phase=phase_s) + + +def _parse_response( + response: httpx.Response, + *, + requested_phase: str, +) -> SidecarAttestResult: + if response.status_code != 200: + if 400 <= response.status_code < 500: + return SidecarAttestMiss( + reason=ConstationFailCode.SIDECAR_RESPONSE_INVALID, + detail=f"http_status={response.status_code}", + ) + return SidecarAttestMiss( + reason=ConstationFailCode.SIDECAR_ATTEST_FAILED, + detail=f"http_status={response.status_code}", + ) + try: + raw: object = response.json() + except ValueError: + return SidecarAttestMiss( + reason=ConstationFailCode.SIDECAR_RESPONSE_INVALID, + detail="body_not_json", + ) + if not isinstance(raw, dict): + return SidecarAttestMiss( + reason=ConstationFailCode.SIDECAR_RESPONSE_INVALID, + detail="body_not_object", + ) + return _parse_wire(raw, requested_phase=requested_phase) + + +def _parse_wire( + wire: Mapping[str, Any], + *, + requested_phase: str, +) -> SidecarAttestResult: + payload_raw = wire.get("payload") + if not isinstance(payload_raw, Mapping): + return SidecarAttestMiss( + reason=ConstationFailCode.SIDECAR_RESPONSE_INVALID, + detail="missing_payload", + ) + + digest = _optional_nonblank_str(payload_raw.get("digest")) + if digest is None: + return SidecarAttestMiss( + reason=ConstationFailCode.SIDECAR_RESPONSE_INVALID, + detail="missing_digest", + ) + nonce = _optional_nonblank_str(payload_raw.get("nonce")) + if nonce is None: + return SidecarAttestMiss( + reason=ConstationFailCode.SIDECAR_RESPONSE_INVALID, + detail="missing_nonce", + ) + pod_id = _optional_nonblank_str(payload_raw.get("pod_id")) + if pod_id is None: + return SidecarAttestMiss( + reason=ConstationFailCode.SIDECAR_RESPONSE_INVALID, + detail="missing_pod_id", + ) + signature = _optional_nonblank_str(wire.get("signature")) + if signature is None: + return SidecarAttestMiss( + reason=ConstationFailCode.SIDECAR_RESPONSE_INVALID, + detail="missing_signature", + ) + + algorithm = _optional_nonblank_str(wire.get("algorithm")) or "hmac-sha256" + schema_version = ( + _optional_nonblank_str(wire.get("schema_version")) + or "prism_attestation_payload.v1" + ) + phase_raw = wire.get("phase", requested_phase) + if isinstance(phase_raw, str): + phase_opt = _optional_nonblank_str(phase_raw) + phase = phase_opt.lower() if phase_opt is not None else requested_phase + else: + phase = requested_phase + if phase not in _VALID_PHASES: + phase = requested_phase + + hashes = _parse_sealed_hashes(payload_raw.get("sealed_manifest_hashes")) + if hashes is None: + return SidecarAttestMiss( + reason=ConstationFailCode.SIDECAR_RESPONSE_INVALID, + detail="invalid_sealed_manifest_hashes", + ) + + return SidecarAttestHit( + digest=digest, + nonce=nonce, + pod_id=pod_id, + phase=phase, + signature=signature, + algorithm=algorithm, + schema_version=schema_version, + sealed_manifest_hashes=hashes, + wire=dict(wire), + ) + + +def _parse_sealed_hashes(raw: object) -> dict[str, str] | None: + if raw is None: + return {} + if not isinstance(raw, Mapping): + return None + out: dict[str, str] = {} + for key, value in raw.items(): + if not isinstance(key, str) or not isinstance(value, str): + return None + path, digest = key.strip(), value.strip() + if not path or not digest: + return None + out[path] = digest + return out + + +def _normalize_phase(phase: str) -> str: + if not isinstance(phase, str): + raise ValueError("phase must be a string") + key = phase.strip().lower() + if key not in _VALID_PHASES: + raise ValueError(f"invalid phase: {phase!r}") + return key + + +def _require_nonblank(field: str, value: str) -> str: + if not isinstance(value, str): + raise ValueError(f"{field} must be a string") + cleaned = value.strip() + if not cleaned: + raise ValueError(f"{field} must be non-empty") + return cleaned + + +def _optional_nonblank_str(value: object) -> str | None: + if not isinstance(value, str): + return None + cleaned = value.strip() + return cleaned if cleaned else None + + +__all__ = [ + "ATTEST_PATH", + "DEFAULT_TIMEOUT_SECONDS", + "HttpSidecarAttestor", + "SidecarAttestHit", + "SidecarAttestMiss", + "SidecarAttestPhase", + "SidecarAttestResult", +] diff --git a/src/base/compute/constation_sidecar_endpoint.py b/src/base/compute/constation_sidecar_endpoint.py new file mode 100644 index 00000000..bdca206f --- /dev/null +++ b/src/base/compute/constation_sidecar_endpoint.py @@ -0,0 +1,197 @@ +"""Derive in-instance sidecar HTTP base URL from a raw Lium pod payload. + +Lium ``GET /pods/{id}`` does not expose a generic HTTP endpoint for custom +container ports. Callers must compose ``executor.executor_ip_address`` with the +published host port from ``ports_mapping`` for the sidecar's internal port. + +This module is pure and fail-closed: it never falls back to ``jupyter_url``, +never parses ``ssh_connect_cmd``, never scans/guesses ports, and never defaults +to the internal port when unmapped. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from enum import StrEnum +from typing import Any, Final + +_MIN_PORT: Final[int] = 1 +_MAX_PORT: Final[int] = 65535 +_HOST_PORT_KEYS: Final[tuple[str, ...]] = ( + "HostPort", + "host_port", + "hostPort", + "port", + "Port", +) + + +class SidecarEndpointMissReason(StrEnum): + """Machine-consumed miss codes for sidecar base URL resolution.""" + + SIDECAR_PORT_UNPUBLISHED = "sidecar_port_unpublished" + POD_ENDPOINT_MISSING = "pod_endpoint_missing" + PORTS_MAPPING_UNPARSEABLE = "ports_mapping_unparseable" + + +@dataclass(frozen=True, slots=True) +class SidecarEndpointHit: + """Resolved sidecar HTTP base URL (no trailing slash).""" + + base_url: str + + def __bool__(self) -> bool: + return True + + +@dataclass(frozen=True, slots=True) +class SidecarEndpointMiss: + """Fail-closed resolution outcome.""" + + reason: SidecarEndpointMissReason + + def __bool__(self) -> bool: + return False + + +SidecarEndpointResult = SidecarEndpointHit | SidecarEndpointMiss + + +def resolve_sidecar_base_url( + pod_raw: Mapping[str, Any], + *, + internal_port: int, + scheme: str = "http", +) -> SidecarEndpointResult: + """Resolve ``{scheme}://{host}:{external_port}`` for the sidecar. + + Parameters + ---------- + pod_raw: + Raw Lium ``PodDetailResponse`` mapping (e.g. ``LiumPodRead.raw``). + internal_port: + Container-internal sidecar listen port to look up in ``ports_mapping``. + scheme: + URL scheme; defaults to ``http``. + """ + host = _extract_host(pod_raw) + if host is None: + return SidecarEndpointMiss(SidecarEndpointMissReason.POD_ENDPOINT_MISSING) + + mapping_result = _normalize_ports_mapping(pod_raw.get("ports_mapping")) + if isinstance(mapping_result, SidecarEndpointMissReason): + return SidecarEndpointMiss(mapping_result) + + external = _lookup_external_port(mapping_result, internal_port) + if external is None: + return SidecarEndpointMiss(SidecarEndpointMissReason.SIDECAR_PORT_UNPUBLISHED) + + return SidecarEndpointHit(base_url=f"{scheme}://{_format_host(host)}:{external}") + + +def _extract_host(pod_raw: Mapping[str, Any]) -> str | None: + executor = pod_raw.get("executor") + if not isinstance(executor, Mapping): + return None + ip = executor.get("executor_ip_address") + if not isinstance(ip, str): + return None + host = ip.strip() + if not host: + return None + return host + + +def _normalize_ports_mapping( + raw: object, +) -> dict[Any, Any] | SidecarEndpointMissReason: + if raw is None: + return {} + if isinstance(raw, Mapping): + return dict(raw) + if isinstance(raw, str): + text = raw.strip() + if not text: + return SidecarEndpointMissReason.PORTS_MAPPING_UNPARSEABLE + try: + parsed: object = json.loads(text) + except json.JSONDecodeError: + return SidecarEndpointMissReason.PORTS_MAPPING_UNPARSEABLE + if not isinstance(parsed, dict): + return SidecarEndpointMissReason.PORTS_MAPPING_UNPARSEABLE + return parsed + return SidecarEndpointMissReason.PORTS_MAPPING_UNPARSEABLE + + +def _lookup_external_port( + mapping: Mapping[Any, Any], + internal_port: int, +) -> int | None: + value = _mapping_get(mapping, internal_port) + if value is None: + return None + return _coerce_external_port(value) + + +def _mapping_get(mapping: Mapping[Any, Any], internal_port: int) -> object | None: + if internal_port in mapping: + return mapping[internal_port] + as_str = str(internal_port) + if as_str in mapping: + return mapping[as_str] + return None + + +def _coerce_external_port(value: object) -> int | None: + direct = _as_port_number(value) + if direct is not None: + return direct + if isinstance(value, Mapping): + return _port_from_hostport_dict(value) + if isinstance(value, list): + for item in value: + port = _coerce_external_port(item) + if port is not None: + return port + return None + return None + + +def _port_from_hostport_dict(value: Mapping[Any, Any]) -> int | None: + for key in _HOST_PORT_KEYS: + if key in value: + port = _as_port_number(value[key]) + if port is not None: + return port + return None + + +def _as_port_number(value: object) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value if _MIN_PORT <= value <= _MAX_PORT else None + if isinstance(value, str): + text = value.strip() + if not text or not text.isdigit(): + return None + port = int(text) + return port if _MIN_PORT <= port <= _MAX_PORT else None + return None + + +def _format_host(host: str) -> str: + if ":" in host and not host.startswith("["): + return f"[{host}]" + return host + + +__all__ = [ + "SidecarEndpointHit", + "SidecarEndpointMiss", + "SidecarEndpointMissReason", + "SidecarEndpointResult", + "resolve_sidecar_base_url", +] diff --git a/src/base/compute/constation_triangle.py b/src/base/compute/constation_triangle.py new file mode 100644 index 00000000..29ba7f51 --- /dev/null +++ b/src/base/compute/constation_triangle.py @@ -0,0 +1,91 @@ +"""Three-way digest triangle — required / Lium-declared / sidecar (fail-closed). + +Compares BASE allowlist **required** digest against Lium **declared** and +sidecar **actual**. Negative-only: any divergence fails with a typed code; +agreement never elevates trust (tamper-evidence only; prism ``constation_ok`` +remains the sole elevation path). +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from base.compute.constation_types import ConstationFailCode + + +def _normalize(value: str | None) -> str | None: + """Strip + casefold; blank becomes None.""" + if value is None: + return None + normalized = str(value).strip().casefold() + if not normalized: + return None + return normalized + + +@dataclass(frozen=True, slots=True) +class DigestTriangleResult: + """Outcome of required / Lium / sidecar digest agreement (ok/fail only).""" + + ok: bool + fail_code: ConstationFailCode | None + + def __bool__(self) -> bool: + return self.ok + + +def evaluate_digest_triangle( + *, + required: str | None, + lium_declared: str | None, + sidecar: str | None, +) -> DigestTriangleResult: + """Fail-closed three-way digest check (precedence order is binding). + + a. required blank/None → REQUIRED_DIGEST_MISMATCH + b. sidecar blank/None → SIDECAR_RESPONSE_INVALID + c. lium_declared blank/None → LIUM_DIGEST_ABSENT + d. sidecar != required → REQUIRED_DIGEST_MISMATCH + e. lium_declared != required → CORROBORATION_MISMATCH + f. otherwise ok=True + """ + req = _normalize(required) + if req is None: + return DigestTriangleResult( + ok=False, + fail_code=ConstationFailCode.REQUIRED_DIGEST_MISMATCH, + ) + + side = _normalize(sidecar) + if side is None: + return DigestTriangleResult( + ok=False, + fail_code=ConstationFailCode.SIDECAR_RESPONSE_INVALID, + ) + + declared = _normalize(lium_declared) + if declared is None: + return DigestTriangleResult( + ok=False, + fail_code=ConstationFailCode.LIUM_DIGEST_ABSENT, + ) + + if side != req: + return DigestTriangleResult( + ok=False, + fail_code=ConstationFailCode.REQUIRED_DIGEST_MISMATCH, + ) + + if declared != req: + return DigestTriangleResult( + ok=False, + fail_code=ConstationFailCode.CORROBORATION_MISMATCH, + ) + + return DigestTriangleResult(ok=True, fail_code=None) + + +__all__ = [ + "DigestTriangleResult", + "evaluate_digest_triangle", +] diff --git a/src/base/compute/constation_types.py b/src/base/compute/constation_types.py index 57139aef..25609d29 100644 --- a/src/base/compute/constation_types.py +++ b/src/base/compute/constation_types.py @@ -33,6 +33,13 @@ class ConstationFailCode(StrEnum): KEY_NOT_REGISTERED = "key_not_registered" SIDECAR_ATTEST_FAILED = "sidecar_attest_failed" RUN_INCOMPLETE = "run_incomplete" + REQUIRED_DIGEST_MISMATCH = "required_digest_mismatch" + LIUM_DIGEST_ABSENT = "lium_digest_absent" + SIDECAR_PORT_UNPUBLISHED = "sidecar_port_unpublished" + POD_HOTKEY_MISMATCH = "pod_hotkey_mismatch" + POD_NOT_RUNNING = "pod_not_running" + SIDECAR_RESPONSE_INVALID = "sidecar_response_invalid" + BUNDLE_INCOMPLETE = "bundle_incomplete" class CorroborationStatus(StrEnum): @@ -57,6 +64,13 @@ class CorroborationStatus(StrEnum): ConstationFailCode.KEY_NOT_REGISTERED: FaultClass.MINER, ConstationFailCode.SIDECAR_ATTEST_FAILED: FaultClass.MINER, ConstationFailCode.RUN_INCOMPLETE: FaultClass.INFRA, + ConstationFailCode.REQUIRED_DIGEST_MISMATCH: FaultClass.MINER, + ConstationFailCode.LIUM_DIGEST_ABSENT: FaultClass.MINER, + ConstationFailCode.SIDECAR_PORT_UNPUBLISHED: FaultClass.MINER, + ConstationFailCode.POD_HOTKEY_MISMATCH: FaultClass.MINER, + ConstationFailCode.POD_NOT_RUNNING: FaultClass.MINER, + ConstationFailCode.SIDECAR_RESPONSE_INVALID: FaultClass.MINER, + ConstationFailCode.BUNDLE_INCOMPLETE: FaultClass.INFRA, } diff --git a/src/base/compute/digest_allowlist.py b/src/base/compute/digest_allowlist.py index 46bb7f9f..4d6d3ef9 100644 --- a/src/base/compute/digest_allowlist.py +++ b/src/base/compute/digest_allowlist.py @@ -24,10 +24,12 @@ from collections.abc import Mapping from dataclasses import dataclass, field from enum import StrEnum +from types import MappingProxyType from typing import Final _FULL_GIT_SHA_RE: Final[re.Pattern[str]] = re.compile(r"^[0-9a-f]{40}$") _IMAGE_DIGEST_RE: Final[re.Pattern[str]] = re.compile(r"^sha256:[0-9a-f]{64}$") +_HEX64_RE: Final[re.Pattern[str]] = re.compile(r"^[0-9a-f]{64}$") class ImageVariant(StrEnum): @@ -88,18 +90,48 @@ def _require_variant(value: ImageVariant | str) -> ImageVariant: ) from exc +def _require_sealed_manifest_hashes( + value: Mapping[str, str], +) -> Mapping[str, str]: + """Validate path → 64-hex lowercase sealed hashes; reject empty maps.""" + if not isinstance(value, Mapping): + raise ValueError("sealed_manifest_hashes must be a mapping") + if not value: + raise ValueError("sealed_manifest_hashes must be non-empty") + out: dict[str, str] = {} + for raw_path, raw_hash in value.items(): + path = str(raw_path).strip() + if not path: + raise ValueError( + "sealed_manifest_hashes path keys must be non-blank strings" + ) + digest = str(raw_hash).strip().lower() + if not _HEX64_RE.fullmatch(digest): + raise ValueError( + f"sealed_manifest_hashes[{path!r}] must be 64-char lowercase hex, " + f"got {raw_hash!r}" + ) + if path in out: + raise ValueError(f"duplicate sealed_manifest_hashes path: {path!r}") + out[path] = digest + return MappingProxyType(out) + + @dataclass(frozen=True, slots=True) class DigestRecord: """One BASE-produced image binding. Keyed conceptually as ``(commit_sha, tree_sha, variant, digest)``. A given - digest maps to exactly one binding. + digest maps to exactly one binding. ``sealed_manifest_hashes`` is the + build-time sealed surface (path → 64-hex SHA-256) prism compares as + ``expected_sealed_manifest_hashes``; must be non-empty at registration. """ commit_sha: str tree_sha: str variant: ImageVariant digest: str + sealed_manifest_hashes: Mapping[str, str] def __post_init__(self) -> None: object.__setattr__( @@ -110,6 +142,11 @@ def __post_init__(self) -> None: ) object.__setattr__(self, "variant", _require_variant(self.variant)) object.__setattr__(self, "digest", _require_image_digest(self.digest)) + object.__setattr__( + self, + "sealed_manifest_hashes", + _require_sealed_manifest_hashes(self.sealed_manifest_hashes), + ) @dataclass(frozen=True, slots=True) diff --git a/src/base/config/settings.py b/src/base/config/settings.py index 5a999f87..5ff097aa 100644 --- a/src/base/config/settings.py +++ b/src/base/config/settings.py @@ -1,5 +1,6 @@ from __future__ import annotations +from pathlib import Path from typing import Literal from pydantic import BaseModel, Field, model_validator @@ -272,6 +273,52 @@ class SecuritySettings(BaseModel): admin_token_file: str | None = None +class ConstationSettings(BaseModel): + """Production constation orchestration (miner Lium key + sidecar attest). + + Fail-closed: ``enabled`` defaults False until ops turns the plane on. + Fernet custody / attestation secrets may be supplied inline or via ``*_file`` + paths (file contents are read lazily by consumers, not here). + Env nesting: ``BASE_CONSTATION__`` (see ``loader._apply_env``). + """ + + enabled: bool = False + custody_master_key: str | None = None + custody_master_key_file: Path | None = None + attestation_verify_key_hex: str | None = None + attestation_build_secret: str | None = None + attestation_build_secret_file: Path | None = None + gap_budget_seconds: float = 30.0 + duration_seconds: float = 300.0 + min_interval_seconds: float = 5.0 + max_interval_seconds: float = 20.0 + max_polls: int = 200 + poll_timeout_seconds: float = 15.0 + sidecar_scheme: str = "http" + sidecar_internal_port: int = 8787 + custody_persist: bool = True + + @model_validator(mode="after") + def validate_constation_bounds(self) -> ConstationSettings: + if self.gap_budget_seconds <= 0: + raise ValueError("gap_budget_seconds must be positive") + if self.duration_seconds <= 0: + raise ValueError("duration_seconds must be positive") + if self.min_interval_seconds <= 0: + raise ValueError("min_interval_seconds must be positive") + if self.max_interval_seconds < self.min_interval_seconds: + raise ValueError("max_interval_seconds must be >= min_interval_seconds") + if self.max_polls <= 0: + raise ValueError("max_polls must be positive") + if self.poll_timeout_seconds <= 0: + raise ValueError("poll_timeout_seconds must be positive") + if not 1 <= self.sidecar_internal_port <= 65535: + raise ValueError("sidecar_internal_port must be in 1..65535") + if self.sidecar_scheme not in {"http", "https"}: + raise ValueError("sidecar_scheme must be 'http' or 'https'") + return self + + class ComputeSettings(BaseModel): """Miner-funded GPU worker plane (architecture.md sec 3.3). @@ -546,6 +593,7 @@ class Settings(BaseModel): docker: DockerSettings = Field(default_factory=DockerSettings) security: SecuritySettings = Field(default_factory=SecuritySettings) compute: ComputeSettings = Field(default_factory=ComputeSettings) + constation: ConstationSettings = Field(default_factory=ConstationSettings) worker: WorkerSettings = Field(default_factory=WorkerSettings) observability: ObservabilitySettings = Field(default_factory=ObservabilitySettings) supervisor: SupervisorSettings = Field(default_factory=SupervisorSettings) diff --git a/src/base/db/models.py b/src/base/db/models.py index ed725f43..ef516028 100644 --- a/src/base/db/models.py +++ b/src/base/db/models.py @@ -1260,6 +1260,12 @@ class ImageDigestAllowlistEntry(Base, TimestampMixin): tree_sha: Mapped[str] = mapped_column(Text, nullable=False) variant: Mapped[str] = mapped_column(Text, nullable=False) digest: Mapped[str] = mapped_column(Text, nullable=False) + sealed_manifest_hashes: Mapped[dict[str, str]] = mapped_column( + JSON, + nullable=False, + server_default="{}", + default=dict, + ) class DeniedImageDigest(Base): diff --git a/src/base/master/constation/__init__.py b/src/base/master/constation/__init__.py index 278a86e8..b85d509c 100644 --- a/src/base/master/constation/__init__.py +++ b/src/base/master/constation/__init__.py @@ -5,6 +5,12 @@ from base.master.constation.allowlist_repository import DigestAllowlistRepository from base.master.constation.bundle_store import ConstationBundleStore from base.master.constation.nonce_repository import DurableAttestationNonceService +from base.master.constation.orchestrator import ( + ConstationOrchestrationRequest, + ConstationOrchestrationResult, + ProductionConstationOrchestrator, +) +from base.master.constation.pod_binding import MinerPodBinding from base.master.constation.routes import ( build_constation_router, create_constation_test_app, @@ -12,8 +18,12 @@ __all__ = [ "ConstationBundleStore", + "ConstationOrchestrationRequest", + "ConstationOrchestrationResult", "DigestAllowlistRepository", "DurableAttestationNonceService", + "MinerPodBinding", + "ProductionConstationOrchestrator", "build_constation_router", "create_constation_test_app", ] diff --git a/src/base/master/constation/allowlist_repository.py b/src/base/master/constation/allowlist_repository.py index 13b92698..15f4afd8 100644 --- a/src/base/master/constation/allowlist_repository.py +++ b/src/base/master/constation/allowlist_repository.py @@ -76,6 +76,7 @@ async def register(self, record: DigestRecord) -> None: tree_sha=row.tree_sha, variant=row.variant, digest=row.digest, + sealed_manifest_hashes=dict(row.sealed_manifest_hashes or {}), ) if bound != record: raise ValueError( @@ -92,6 +93,7 @@ async def register(self, record: DigestRecord) -> None: tree_sha=record.tree_sha, variant=record.variant.value, digest=record.digest, + sealed_manifest_hashes=dict(record.sealed_manifest_hashes), ) ) @@ -140,6 +142,7 @@ async def load_allowlist(self) -> DigestAllowlist: tree_sha=row.tree_sha, variant=ImageVariant(row.variant), digest=row.digest, + sealed_manifest_hashes=dict(row.sealed_manifest_hashes or {}), ) ) for row in denied_d: diff --git a/src/base/master/constation/attestation_keys.py b/src/base/master/constation/attestation_keys.py new file mode 100644 index 00000000..45bcbabc --- /dev/null +++ b/src/base/master/constation/attestation_keys.py @@ -0,0 +1,34 @@ +"""Load BASE-held attestation verify key from settings (fail-closed).""" + +from __future__ import annotations + +from typing import Protocol + + +class _HasConstationVerifyKey(Protocol): + """Minimal settings surface for verify-key load (avoids full Settings import).""" + + @property + def constation(self) -> _ConstationKeyHex: ... + + +class _ConstationKeyHex(Protocol): + attestation_verify_key_hex: str | None + + +def load_attestation_verify_key(settings: _HasConstationVerifyKey) -> bytes | None: + """Parse ``settings.constation.attestation_verify_key_hex`` to raw key bytes. + + Empty / missing / whitespace-only → ``None`` (router returns ``empty_key``). + Non-empty hex is decoded with ``bytes.fromhex`` (invalid hex raises). + """ + raw = settings.constation.attestation_verify_key_hex + if raw is None: + return None + text = raw.strip() + if not text: + return None + return bytes.fromhex(text) + + +__all__ = ["load_attestation_verify_key"] diff --git a/src/base/master/constation/bundle_seal.py b/src/base/master/constation/bundle_seal.py new file mode 100644 index 00000000..b975e1c4 --- /dev/null +++ b/src/base/master/constation/bundle_seal.py @@ -0,0 +1,132 @@ +"""Pure sealer: assemble prism ``ConstationBundle`` wire dict (no I/O). + +Does **not** issue or consume nonces. Callers pass the end-phase nonce and the +last good sidecar signed wire; this module only maps allowlist + run record + +those inputs onto the prism wire field names. +""" + +from __future__ import annotations + +import copy +from collections.abc import Mapping +from typing import Any, Final + +from base.compute.constation_types import ConstationRunRecord +from base.compute.digest_allowlist import DigestRecord + +_BUNDLE_INCOMPLETE: Final[str] = "BUNDLE_INCOMPLETE" + + +def seal_constation_bundle( + *, + allowlist_record: DigestRecord, + run_record: ConstationRunRecord, + nonce: str, + signed_attestation: Mapping[str, Any] | None, +) -> dict[str, object]: + """Build a prism-compatible constation bundle wire dict. + + Field sources: + + * Identity (``commit_sha``, ``tree_sha``, ``variant``, ``digest``, + ``expected_sealed_manifest_hashes``) — ``allowlist_record`` + * Binding (``work_unit_id``, ``miner_hotkey``, ``pod_id``) — ``run_record`` + * ``nonce`` — caller-supplied end-phase nonce (not issued here) + * ``signed_attestation`` — last good sidecar answer wire (opaque object) + * ``reported_sealed_manifest_hashes`` — from sidecar wire payload when + present and non-empty; otherwise a copy of expected + * ``lium_declared_digest``, gap budget/observed — ``run_record`` + + Raises: + ValueError: fail-closed when a required field is missing/blank + (message includes ``BUNDLE_INCOMPLETE``). + """ + nonce_s = _require_nonblank("nonce", nonce) + work_unit_id = _require_nonblank("work_unit_id", run_record.work_unit_id) + miner_hotkey = _require_nonblank("miner_hotkey", run_record.miner_hotkey) + pod_id = _require_nonblank("pod_id", run_record.pod_id) + + if signed_attestation is None: + raise ValueError(f"{_BUNDLE_INCOMPLETE}: missing signed_attestation") + if not isinstance(signed_attestation, Mapping): + raise ValueError( + f"{_BUNDLE_INCOMPLETE}: signed_attestation must be a mapping wire object" + ) + + expected = { + str(path): str(digest) + for path, digest in allowlist_record.sealed_manifest_hashes.items() + } + if not expected: + raise ValueError( + f"{_BUNDLE_INCOMPLETE}: expected_sealed_manifest_hashes must be non-empty" + ) + + reported = _reported_sealed_manifest_hashes(signed_attestation, expected) + wire_attestation: dict[str, object] = copy.deepcopy(dict(signed_attestation)) + + variant = allowlist_record.variant + variant_s = variant.value if hasattr(variant, "value") else str(variant) + + return { + "commit_sha": str(allowlist_record.commit_sha), + "tree_sha": str(allowlist_record.tree_sha), + "variant": variant_s, + "digest": str(allowlist_record.digest), + "work_unit_id": work_unit_id, + "miner_hotkey": miner_hotkey, + "pod_id": pod_id, + "nonce": nonce_s, + "signed_attestation": wire_attestation, + "expected_sealed_manifest_hashes": dict(expected), + "reported_sealed_manifest_hashes": dict(reported), + "lium_declared_digest": run_record.lium_declared_digest, + "constation_gap_budget_seconds": float( + run_record.constation_gap_budget_seconds + ), + "constation_observed_max_gap_seconds": float( + run_record.constation_observed_max_gap_seconds + ), + } + + +def _require_nonblank(name: str, value: str) -> str: + if not isinstance(value, str): + raise ValueError(f"{_BUNDLE_INCOMPLETE}: {name} must be a non-empty string") + stripped = value.strip() + if not stripped: + raise ValueError(f"{_BUNDLE_INCOMPLETE}: {name} must be a non-empty string") + return stripped + + +def _reported_sealed_manifest_hashes( + wire: Mapping[str, Any], + expected: Mapping[str, str], +) -> dict[str, str]: + """Prefer sidecar payload hashes; fall back to expected allowlist surface.""" + raw: object | None = None + payload = wire.get("payload") + if isinstance(payload, Mapping): + raw = payload.get("sealed_manifest_hashes") + if raw is None: + raw = wire.get("sealed_manifest_hashes") + parsed = _as_str_str_map(raw) + if parsed: + return parsed + return dict(expected) + + +def _as_str_str_map(raw: object) -> dict[str, str] | None: + if not isinstance(raw, Mapping) or not raw: + return None + out: dict[str, str] = {} + for key, value in raw.items(): + path = str(key).strip() + digest = str(value).strip() + if not path or not digest: + return None + out[path] = digest + return out or None + + +__all__ = ["seal_constation_bundle"] diff --git a/src/base/master/constation/custody_keys.py b/src/base/master/constation/custody_keys.py new file mode 100644 index 00000000..25093fac --- /dev/null +++ b/src/base/master/constation/custody_keys.py @@ -0,0 +1,215 @@ +"""Load custody master key and build production constation runtime (fail-closed).""" + +from __future__ import annotations + +import asyncio +import logging +import random +import time +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol + +from base.compute.constation_custody import LiumKeyCustody +from base.compute.constation_poller import PollerConfig +from base.master.constation.orchestrator import ProductionConstationOrchestrator +from base.master.constation.pod_binding import MinerPodBinding +from base.security.admin_auth import read_secret + +logger = logging.getLogger(__name__) + + +class _HasConstation(Protocol): + @property + def constation(self) -> _ConstationSurface: ... + + +class _ConstationSurface(Protocol): + enabled: bool + custody_master_key: str | None + custody_master_key_file: Path | None + gap_budget_seconds: float + min_interval_seconds: float + max_interval_seconds: float + max_polls: int + sidecar_internal_port: int + poll_timeout_seconds: float + + +@dataclass(frozen=True, slots=True) +class ConstationRuntime: + """Optional production constation services attached at master boot.""" + + enabled: bool + pod_binding: MinerPodBinding | None + orchestrator: ProductionConstationOrchestrator | None + + +def load_custody_master_key(settings: _HasConstation) -> bytes | None: + """Read Fernet master key from settings (inline or file). Empty → None.""" + cs = settings.constation + file_path = cs.custody_master_key_file + raw = read_secret( + cs.custody_master_key, + str(file_path) if file_path is not None else None, + ) + text = raw.strip() if raw else "" + if not text: + return None + return text.encode("utf-8") + + +def poller_config_from_settings(cs: _ConstationSurface) -> PollerConfig: + """Map ConstationSettings poll fields onto PollerConfig.""" + max_polls = int(cs.max_polls) + return PollerConfig( + gap_budget_seconds=float(cs.gap_budget_seconds), + min_interval_seconds=float(cs.min_interval_seconds), + max_interval_seconds=float(cs.max_interval_seconds), + max_polls=max_polls, + max_cost_units=float(max_polls), + ) + + +def build_constation_runtime( + settings: _HasConstation, + *, + nonce_service: Any, + bundle_store: Any, +) -> ConstationRuntime: + """Construct custody + binding + orchestrator when enabled and key present. + + Fail-closed: enabled without a usable master key logs an error and returns + ``pod_binding=None`` / ``orchestrator=None`` so master boot continues and + register_miner_key stays 503. Never logs key material. + """ + cs = settings.constation + if not cs.enabled: + return ConstationRuntime(enabled=False, pod_binding=None, orchestrator=None) + + master_key = load_custody_master_key(settings) + if master_key is None: + logger.error( + "constation.enabled is True but custody master key is missing " + "(set custody_master_key or custody_master_key_file); " + "constation custody/orchestrator disabled" + ) + return ConstationRuntime(enabled=True, pod_binding=None, orchestrator=None) + + try: + custody = LiumKeyCustody(master_key=master_key) + except (ValueError, TypeError) as exc: + logger.error( + "constation.enabled is True but custody master key is invalid " + "(%s); constation custody/orchestrator disabled", + type(exc).__name__, + ) + return ConstationRuntime(enabled=True, pod_binding=None, orchestrator=None) + + pod_binding = MinerPodBinding(custody=custody) + orchestrator = ProductionConstationOrchestrator( + pod_binding=pod_binding, + nonce_service=nonce_service, + bundle_store=bundle_store, + poller_config=poller_config_from_settings(cs), + now_fn=time.monotonic, + sleep_fn=asyncio.sleep, + rng_fn=random.random, + sidecar_internal_port=int(cs.sidecar_internal_port), + sidecar_timeout_seconds=float(cs.poll_timeout_seconds), + ) + return ConstationRuntime( + enabled=True, pod_binding=pod_binding, orchestrator=orchestrator + ) + + +def make_constation_pre_forward_hook( + orchestrator: ProductionConstationOrchestrator | None, + *, + duration_seconds: float, +): + """Return async hook for WorkerReconciliationService (or None). + + Invokes orchestrator only when work-unit metadata carries full identity + (required_digest, commit/tree/variant, sealed_manifest_hashes). Incomplete + identity is skipped with a debug log — services remain wired for register. + Hook errors are logged and do not block result forward. + """ + if orchestrator is None: + return None + + async def _hook( + *, + work_unit_id: str, + miner_hotkey: str, + metadata: Mapping[str, Any], + ) -> None: + from base.master.constation.orchestrator import ( + ConstationOrchestrationRequest, + ) + + md = dict(metadata or {}) + required_digest = md.get("required_digest") or md.get("digest") + commit_sha = md.get("commit_sha") + tree_sha = md.get("tree_sha") + variant = md.get("variant") + sealed = md.get("sealed_manifest_hashes") + if not ( + isinstance(required_digest, str) + and required_digest + and isinstance(commit_sha, str) + and commit_sha + and isinstance(tree_sha, str) + and tree_sha + and variant is not None + and isinstance(sealed, Mapping) + and sealed + ): + logger.debug( + "constation pre-forward hook skipped: incomplete identity " + "on work_unit_id=%s miner_hotkey=%s", + work_unit_id, + miner_hotkey, + ) + return + try: + await orchestrator.run( + ConstationOrchestrationRequest( + work_unit_id=work_unit_id, + miner_hotkey=miner_hotkey, + required_digest=required_digest, + commit_sha=commit_sha, + tree_sha=tree_sha, + variant=variant, + sealed_manifest_hashes=dict(sealed), + duration_seconds=float( + md.get("duration_seconds", duration_seconds) + ), + pod_id=md.get("pod_id") + if isinstance(md.get("pod_id"), str) + else None, + instance_id=( + md.get("instance_id") + if isinstance(md.get("instance_id"), str) + else None + ), + ) + ) + except Exception: + logger.exception( + "constation orchestrator failed for work_unit_id=%s " + "(forward continues)", + work_unit_id, + ) + + return _hook + + +__all__ = [ + "ConstationRuntime", + "build_constation_runtime", + "load_custody_master_key", + "make_constation_pre_forward_hook", + "poller_config_from_settings", +] diff --git a/src/base/master/constation/orchestrator.py b/src/base/master/constation/orchestrator.py new file mode 100644 index 00000000..12b9b40a --- /dev/null +++ b/src/base/master/constation/orchestrator.py @@ -0,0 +1,344 @@ +"""Production constation orchestrator: issue → run → seal → put (never consume). + +B2 +-- +Nonces are single-use and prism consumes at ingest. This orchestrator **issues** +nonces for each poll (via ``poll_nonce_fn``) and seals the end-phase nonce into +the bundle. It **never** calls ``consume`` — the end-phase nonce must remain +first-consumable after ``bundle_store.put``. +""" + +from __future__ import annotations + +import asyncio +import inspect +import logging +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass +from typing import Any, Protocol + +from base.compute.attestation_nonce import NonceBinding, NonceRecord +from base.compute.constation_poller import PollerConfig +from base.compute.constation_runner import ( + AttestorFactory, + ConstationRunner, + ConstationRunRequest, + NowFn, + RngFn, + SidecarAttestor, + SleepFn, +) +from base.compute.constation_sidecar_client import DEFAULT_TIMEOUT_SECONDS +from base.compute.constation_types import ( + ConstationFailCode, + ConstationRunRecord, +) +from base.compute.digest_allowlist import DigestRecord, ImageVariant +from base.master.constation.bundle_seal import seal_constation_bundle +from base.master.constation.bundle_store import ConstationBundleStore +from base.master.constation.pod_binding import MinerPodBinding + +logger = logging.getLogger(__name__) + +RunnerFactory = Callable[..., Any] + + +class NonceIssuer(Protocol): + """Sync or async nonce issuer (in-memory or durable). Never consume here.""" + + def issue(self, binding: NonceBinding) -> NonceRecord | Awaitable[NonceRecord]: ... + + +@dataclass(frozen=True, slots=True) +class ConstationOrchestrationRequest: + """Explicit inputs for one production constation run (no hidden globals).""" + + work_unit_id: str + miner_hotkey: str + required_digest: str + commit_sha: str + tree_sha: str + variant: ImageVariant | str + sealed_manifest_hashes: Mapping[str, str] + duration_seconds: float + pod_id: str | None = None + instance_id: str | None = None + + +@dataclass(frozen=True, slots=True) +class ConstationOrchestrationResult: + """Outcome of :meth:`ProductionConstationOrchestrator.run`.""" + + ok: bool + reason: ConstationFailCode + run_record: ConstationRunRecord | None + bundle: dict[str, object] | None + end_phase_nonce: str | None + + +@dataclass +class ProductionConstationOrchestrator: + """Issue nonces → ConstationRunner → seal_constation_bundle → bundle_store.put. + + Caller gates on settings (constation enabled). This class is fail-closed and + unit-testable via ``runner_factory`` and fakes for nonce/binding/store. + """ + + pod_binding: MinerPodBinding + nonce_service: NonceIssuer + bundle_store: ConstationBundleStore + poller_config: PollerConfig + now_fn: NowFn + sleep_fn: SleepFn + rng_fn: RngFn + sidecar: SidecarAttestor | None = None + attestor_factory: AttestorFactory | None = None + sidecar_internal_port: int | None = None + sidecar_timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS + runner_factory: RunnerFactory | None = None + + async def run( + self, request: ConstationOrchestrationRequest + ) -> ConstationOrchestrationResult: + """Execute one constation orchestration; never consume nonces.""" + hotkey = request.miner_hotkey.strip() + work_unit_id = request.work_unit_id.strip() + + resolved = self._resolve_pod_id(request, hotkey) + if resolved is None: + logger.warning( + "constation orchestrator: no binding for miner_hotkey=%s", + hotkey, + ) + return ConstationOrchestrationResult( + ok=False, + reason=ConstationFailCode.KEY_NOT_REGISTERED, + run_record=None, + bundle=None, + end_phase_nonce=None, + ) + pod_id = resolved + + if not self.pod_binding.custody.has_key(hotkey): + return ConstationOrchestrationResult( + ok=False, + reason=ConstationFailCode.KEY_NOT_REGISTERED, + run_record=None, + bundle=None, + end_phase_nonce=None, + ) + + binding = NonceBinding( + work_unit_id=work_unit_id, + miner_hotkey=hotkey, + pod_id=pod_id, + ) + issued: list[str] = [] + + def poll_nonce_fn() -> str: + """Issue-only poll nonce for ConstationRunner (sync NonceFn). + + Supports sync ``AttestationNonceService.issue`` and async + ``DurableAttestationNonceService.issue`` (via a worker-thread loop + so we never deadlock the running event loop). Never consumes. + """ + record = _issue_blocking(self.nonce_service, binding) + issued.append(record.nonce) + return record.nonce + + runner = self._build_runner(poll_nonce_fn=poll_nonce_fn) + run_req = ConstationRunRequest( + miner_hotkey=hotkey, + work_unit_id=work_unit_id, + pod_id=pod_id, + duration_seconds=request.duration_seconds, + required_digest=request.required_digest.strip(), + ) + run_record = await runner.run(run_req) + + if not run_record.ok: + return ConstationOrchestrationResult( + ok=False, + reason=run_record.reason, + run_record=run_record, + bundle=None, + end_phase_nonce=None, + ) + + end_nonce = issued[-1] if issued else None + if end_nonce is None: + # Runner paths that never dial HttpSidecarAttestor still need an + # end-phase nonce for the sealed bundle (issue-only). + try: + end_record = await _maybe_await_issue(self.nonce_service.issue(binding)) + except Exception as exc: + logger.warning( + "constation orchestrator: end-phase issue failed wu=%s err=%s", + work_unit_id, + type(exc).__name__, + ) + return ConstationOrchestrationResult( + ok=False, + reason=ConstationFailCode.RUN_INCOMPLETE, + run_record=run_record, + bundle=None, + end_phase_nonce=None, + ) + end_nonce = end_record.nonce + issued.append(end_nonce) + + wire = getattr(runner, "last_signed_wire", None) + if wire is None: + logger.warning( + "constation orchestrator: missing last_signed_wire wu=%s", + work_unit_id, + ) + return ConstationOrchestrationResult( + ok=False, + reason=ConstationFailCode.RUN_INCOMPLETE, + run_record=run_record, + bundle=None, + end_phase_nonce=None, + ) + + try: + variant = ( + request.variant + if isinstance(request.variant, ImageVariant) + else ImageVariant(str(request.variant).strip().lower()) + ) + allowlist_record = DigestRecord( + commit_sha=request.commit_sha, + tree_sha=request.tree_sha, + variant=variant, + digest=request.required_digest, + sealed_manifest_hashes=dict(request.sealed_manifest_hashes), + ) + bundle = seal_constation_bundle( + allowlist_record=allowlist_record, + run_record=run_record, + nonce=end_nonce, + signed_attestation=wire, + ) + except ValueError as exc: + logger.warning( + "constation orchestrator: seal failed wu=%s err=%s", + work_unit_id, + exc, + ) + return ConstationOrchestrationResult( + ok=False, + reason=ConstationFailCode.RUN_INCOMPLETE, + run_record=run_record, + bundle=None, + end_phase_nonce=None, + ) + + self.bundle_store.put(work_unit_id, bundle) + return ConstationOrchestrationResult( + ok=True, + reason=ConstationFailCode.OK, + run_record=run_record, + bundle=bundle, + end_phase_nonce=end_nonce, + ) + + def _resolve_pod_id( + self, + request: ConstationOrchestrationRequest, + hotkey: str, + ) -> str | None: + if request.pod_id is not None and str(request.pod_id).strip(): + return str(request.pod_id).strip() + if request.instance_id is not None and str(request.instance_id).strip(): + return str(request.instance_id).strip() + if not self.pod_binding.has_binding(hotkey): + return None + instance = self.pod_binding.get_instance_id(hotkey) + if instance is None or not instance.strip(): + return None + return instance.strip() + + def _build_runner(self, *, poll_nonce_fn: Callable[[], str]) -> Any: + factory = self.runner_factory + if factory is not None: + return factory( + custody=self.pod_binding.custody, + sidecar=self.sidecar if self.sidecar is not None else _NullSidecar(), + poller_config=self.poller_config, + now_fn=self.now_fn, + sleep_fn=self.sleep_fn, + rng_fn=self.rng_fn, + attestor_factory=self.attestor_factory, + sidecar_internal_port=self.sidecar_internal_port, + sidecar_timeout_seconds=self.sidecar_timeout_seconds, + poll_nonce_fn=poll_nonce_fn, + ) + sidecar: SidecarAttestor + if self.sidecar is not None: + sidecar = self.sidecar + else: + sidecar = _NullSidecar() + return ConstationRunner( + custody=self.pod_binding.custody, + sidecar=sidecar, + poller_config=self.poller_config, + now_fn=self.now_fn, + sleep_fn=self.sleep_fn, + rng_fn=self.rng_fn, + attestor_factory=self.attestor_factory, + sidecar_internal_port=self.sidecar_internal_port, + sidecar_timeout_seconds=self.sidecar_timeout_seconds, + poll_nonce_fn=poll_nonce_fn, + ) + + +@dataclass +class _NullSidecar: + """Placeholder when factory/port supplies the real attestor.""" + + async def attest(self, *, pod_id: str, phase: str) -> str: + del pod_id, phase + raise RuntimeError("sidecar not configured") + + +async def _maybe_await_issue( + value: NonceRecord | Awaitable[NonceRecord], +) -> NonceRecord: + if inspect.isawaitable(value): + return await value + return value + + +def _issue_blocking(nonce_service: NonceIssuer, binding: NonceBinding) -> NonceRecord: + """Call ``issue`` from a sync context (runner poll_nonce_fn).""" + issue = nonce_service.issue + # Coroutine function (DurableAttestationNonceService.issue) + if inspect.iscoroutinefunction(issue): + return _run_coro_in_worker(issue(binding)) + record = issue(binding) + if inspect.isawaitable(record): + return _run_coro_in_worker(record) + return record + + +def _run_coro_in_worker(coro: Awaitable[NonceRecord]) -> NonceRecord: + """Run ``coro`` on a fresh loop in a worker thread (no same-loop deadlock).""" + import concurrent.futures + + async def _await_once() -> NonceRecord: + return await coro + + def _thread_main() -> NonceRecord: + return asyncio.run(_await_once()) + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(_thread_main).result(timeout=60.0) + + +__all__ = [ + "ConstationOrchestrationRequest", + "ConstationOrchestrationResult", + "NonceIssuer", + "ProductionConstationOrchestrator", +] diff --git a/src/base/master/constation/pod_binding.py b/src/base/master/constation/pod_binding.py new file mode 100644 index 00000000..36da7997 --- /dev/null +++ b/src/base/master/constation/pod_binding.py @@ -0,0 +1,167 @@ +"""Miner Lium API key + instance_id binding (domain; no HTTP). + +Registration is fail-closed: + +1. Probe the API key (same path as ``LiumKeyCustody``) +2. ``get_pod_raw(instance_id)`` via the probed client +3. :func:`~base.compute.constation_pod.assert_pod_bound` (running + hotkey match) +4. Only then store encrypted key + ``instance_id`` keyed by miner hotkey + +Never logs ``api_key``. Routes live in a later task. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field + +from base.compute.constation_custody import LiumKeyCustody +from base.compute.constation_pod import assert_pod_bound +from base.compute.constation_types import ConstationFailCode, ConstationVerdict +from base.compute.lium import ( + LiumAuthError, + LiumError, + LiumNotFoundError, + LiumRateLimitError, +) + +logger = logging.getLogger(__name__) + + +@dataclass +class MinerPodBinding: + """In-memory miner hotkey → (encrypted Lium key, instance_id) binding. + + Composes :class:`LiumKeyCustody` for Fernet key storage. Instance ids are + plain strings (not secret). ``repr`` never includes api keys. + """ + + custody: LiumKeyCustody + _instance_by_hotkey: dict[str, str] = field( + default_factory=dict, init=False, repr=False + ) + + def __repr__(self) -> str: + return f"MinerPodBinding(bound={len(self._instance_by_hotkey)})" + + def has_binding(self, miner_hotkey: str) -> bool: + hotkey = miner_hotkey.strip() + return hotkey in self._instance_by_hotkey and self.custody.has_key(hotkey) + + def get_instance_id(self, miner_hotkey: str) -> str | None: + return self._instance_by_hotkey.get(miner_hotkey.strip()) + + async def register( + self, + *, + miner_hotkey: str, + api_key: str, + instance_id: str, + ) -> ConstationVerdict: + """Probe key, bind pod, then store encrypted key + instance_id. + + Fail closed on bad key / mismatch / not running / pod fetch errors. + Never logs ``api_key``. + """ + hotkey = _require_nonblank("miner_hotkey", miner_hotkey) + key = _require_nonblank("api_key", api_key) + pod_id = _require_nonblank("instance_id", instance_id) + + client = self.custody.client_factory(key) + try: + await self.custody.probe_fn(client) + except LiumAuthError: + logger.warning("lium key probe rejected (401) for miner_hotkey=%s", hotkey) + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.LIUM_AUTH_REVOKED, + detail="probe_401", + ) + except LiumError as exc: + logger.warning( + "lium key probe failed for miner_hotkey=%s status=%s", + hotkey, + getattr(exc, "status_code", None), + ) + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.PROBE_FAILED, + detail=type(exc).__name__, + ) + + try: + pod = await client.get_pod_raw(pod_id) + except LiumAuthError: + logger.warning( + "lium get_pod rejected (401) for miner_hotkey=%s instance_id=%s", + hotkey, + pod_id, + ) + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.LIUM_AUTH_REVOKED, + detail="get_pod_401", + ) + except LiumNotFoundError: + logger.warning( + "lium get_pod not found for miner_hotkey=%s instance_id=%s", + hotkey, + pod_id, + ) + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.POD_HOTKEY_MISMATCH, + detail="pod_not_found", + ) + except LiumRateLimitError: + logger.warning( + "lium get_pod rate limited for miner_hotkey=%s instance_id=%s", + hotkey, + pod_id, + ) + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.LIUM_RATE_LIMITED, + detail="get_pod_429", + ) + except LiumError as exc: + logger.warning( + "lium get_pod failed for miner_hotkey=%s instance_id=%s status=%s", + hotkey, + pod_id, + getattr(exc, "status_code", None), + ) + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.PROBE_FAILED, + detail=type(exc).__name__, + ) + + bound = assert_pod_bound(pod_raw=pod.raw, expected_hotkey=hotkey) + if not bound.ok: + logger.warning( + "pod bind failed for miner_hotkey=%s instance_id=%s reason=%s", + hotkey, + pod_id, + bound.reason, + ) + return bound + + self.custody.store_probed_key(miner_hotkey=hotkey, api_key=key) + self._instance_by_hotkey[hotkey] = pod_id + logger.info( + "miner pod bound for miner_hotkey=%s instance_id=%s", + hotkey, + pod_id, + ) + return ConstationVerdict(ok=True, reason=ConstationFailCode.OK) + + +def _require_nonblank(field_name: str, value: str) -> str: + normalized = value.strip() + if not normalized: + raise ValueError(f"{field_name} must be a non-empty string, got {value!r}") + return normalized + + +__all__ = ["MinerPodBinding"] diff --git a/src/base/master/constation/routes.py b/src/base/master/constation/routes.py index 08f29c8b..d747d4f4 100644 --- a/src/base/master/constation/routes.py +++ b/src/base/master/constation/routes.py @@ -8,6 +8,7 @@ POST /internal/v1/constation/check_allowlist POST /internal/v1/constation/check_nonce POST /internal/v1/constation/register_digest + POST /internal/v1/constation/register_miner_key POST /internal/v1/constation/verify_attestation PUT /internal/v1/constation/bundle/{work_unit_id} GET /internal/v1/constation/bundle/{work_unit_id} @@ -32,6 +33,7 @@ from base.master.constation.allowlist_repository import DigestAllowlistRepository from base.master.constation.bundle_store import ConstationBundleStore from base.master.constation.nonce_repository import DurableAttestationNonceService +from base.master.constation.pod_binding import MinerPodBinding class RegisterDigestBody(BaseModel): @@ -39,6 +41,7 @@ class RegisterDigestBody(BaseModel): tree_sha: str variant: str digest: str + sealed_manifest_hashes: dict[str, str] class CheckAllowlistBody(BaseModel): @@ -67,6 +70,12 @@ class VerifyAttestationBody(BaseModel): key_hex: str | None = None +class RegisterMinerKeyBody(BaseModel): + miner_hotkey: str + api_key: str + instance_id: str + + class AnswerBody(BaseModel): model_config = {"extra": "allow"} @@ -121,6 +130,7 @@ def build_constation_router( internal_token: str | None = None, attestation_verify_key: bytes | None = None, default_binding: NonceBinding | None = None, + pod_binding: MinerPodBinding | None = None, ) -> APIRouter: """Build router; require Bearer when ``internal_token`` is set.""" store = bundle_store or ConstationBundleStore() @@ -213,15 +223,51 @@ async def issue_nonce(body: IssueNonceBody) -> dict[str, Any]: dependencies=[Depends(require_internal)], ) async def register_digest(body: RegisterDigestBody) -> dict[str, str]: - record = DigestRecord( - commit_sha=body.commit_sha, - tree_sha=body.tree_sha, - variant=ImageVariant(body.variant.strip().lower()), - digest=body.digest, - ) - await allowlist_repo.register(record) + try: + record = DigestRecord( + commit_sha=body.commit_sha, + tree_sha=body.tree_sha, + variant=ImageVariant(body.variant.strip().lower()), + digest=body.digest, + sealed_manifest_hashes=body.sealed_manifest_hashes, + ) + await allowlist_repo.register(record) + except ValueError as exc: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=str(exc), + ) from exc return {"status": "registered", "digest": record.digest} + @router.post( + "/internal/v1/constation/register_miner_key", + dependencies=[Depends(require_internal)], + ) + async def register_miner_key(body: RegisterMinerKeyBody) -> dict[str, str]: + """Bind miner Lium API key + instance_id after probe/pod checks.""" + if pod_binding is None: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + detail="miner pod binding not configured", + ) + try: + verdict = await pod_binding.register( + miner_hotkey=body.miner_hotkey, + api_key=body.api_key, + instance_id=body.instance_id, + ) + except ValueError as exc: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=str(exc), + ) from exc + if not verdict.ok: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=verdict.reason.value, + ) + return {"status": "registered"} + @router.post( "/internal/v1/constation/check_allowlist", dependencies=[Depends(require_internal)], @@ -312,6 +358,7 @@ def create_constation_test_app( default_binding: NonceBinding | None = None, attestation_verify_key: bytes | None = None, bundle_store: ConstationBundleStore | None = None, + pod_binding: MinerPodBinding | None = None, ) -> Any: """Minimal FastAPI app hosting only the constation router (unit tests).""" from fastapi import FastAPI @@ -324,6 +371,7 @@ def create_constation_test_app( internal_token=internal_token, default_binding=default_binding, attestation_verify_key=attestation_verify_key, + pod_binding=pod_binding, ) app.include_router(router) app.state.constation_router = router diff --git a/src/base/master/worker_reconciliation.py b/src/base/master/worker_reconciliation.py index 43a2f283..d7739a3d 100644 --- a/src/base/master/worker_reconciliation.py +++ b/src/base/master/worker_reconciliation.py @@ -122,6 +122,22 @@ async def forward_result( ) -> None: ... +class ConstationPreForwardHook(Protocol): + """Optional pre-forward hook (e.g. production constation orchestrator). + + Invoked in :meth:`WorkerReconciliationService._forward` before + ``forward_result`` when identity metadata may be available on the unit. + """ + + async def __call__( + self, + *, + work_unit_id: str, + miner_hotkey: str, + metadata: Mapping[str, Any], + ) -> None: ... + + @dataclass(frozen=True) class ReconciliationPassResult: """Observable outcome of one reconciliation pass. @@ -150,11 +166,13 @@ def __init__( result_forwarder: ChallengeResultForwarder | None = None, required_capability: str = CAPABILITY_GPU, now_fn: Callable[[], datetime] = lambda: datetime.now(UTC), + constation_hook: ConstationPreForwardHook | None = None, ) -> None: self._session_factory = session_factory self._result_forwarder = result_forwarder self._required_capability = required_capability self._now_fn = now_fn + self._constation_hook = constation_hook def transaction(self) -> AbstractAsyncContextManager[AsyncSession]: """Open a single committed transaction over the control-plane DB.""" @@ -381,6 +399,19 @@ async def _ensure_audit_unit( return audit_id async def _forward(self, primary: WorkAssignment, winner: WorkerAssignment) -> bool: + if self._constation_hook is not None: + try: + await self._constation_hook( + work_unit_id=primary.work_unit_id, + miner_hotkey=str(winner.miner_hotkey or ""), + metadata=dict(primary.payload or {}), + ) + except Exception: + logger.exception( + "constation pre-forward hook failed for unit %s " + "(forward continues)", + primary.work_unit_id, + ) if self._result_forwarder is None: return True try: @@ -405,6 +436,7 @@ async def _forward(self, primary: WorkAssignment, winner: WorkerAssignment) -> b "AUDIT_WORK_UNIT_SUFFIX", "RECONCILE_DEGRADED_PAYLOAD_KEY", "ChallengeResultForwarder", + "ConstationPreForwardHook", "ReconciliationPassResult", "WorkerReconciliationService", "audit_work_unit_id", diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/surface/__init__.py b/tests/surface/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/surface/constation_surface_helpers.py b/tests/surface/constation_surface_helpers.py new file mode 100644 index 00000000..b2486f89 --- /dev/null +++ b/tests/surface/constation_surface_helpers.py @@ -0,0 +1,271 @@ +"""Shared fixtures/helpers for production constation surface tests (T12).""" + +from __future__ import annotations + +import logging +import sys +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import httpx + +from base.compute.attestation_nonce import AttestationNonceService +from base.compute.constation_custody import LiumKeyCustody, generate_custody_master_key +from base.compute.constation_poller import PollerConfig +from base.compute.constation_runner import ConstationRunRequest +from base.compute.constation_types import ( + ConstationFailCode, + ConstationRunRecord, + CorroborationStatus, + FaultClass, +) +from base.compute.digest_allowlist import ImageVariant +from base.master.constation.bundle_store import ConstationBundleStore +from base.master.constation.orchestrator import ( + ConstationOrchestrationRequest, + ProductionConstationOrchestrator, +) +from base.master.constation.pod_binding import MinerPodBinding + +COMMIT = "a" * 40 +TREE = "b" * 40 +DIGEST = "sha256:" + ("c" * 64) +DIGEST_BAD = "sha256:" + ("d" * 64) +MANIFEST = {"src/harness.py": "e" * 64} +HOTKEY = "5MinerSurfaceTestHotkey000000000000001" +POD = "pod-surface-001" +WORK_UNIT = "wu-surface-001" +T0 = datetime(2026, 7, 27, 5, 0, 0, tzinfo=UTC) +TOKEN = "surface-internal" +BUILD_SECRET = b"surface-build-secret-fixture" + +WIRE: dict[str, Any] = { + "payload": { + "digest": DIGEST, + "nonce": "will-be-overwritten", + "pod_id": POD, + "variant": "cuda", + "build_secret_response": "ab" * 32, + "sealed_manifest_hashes": dict(MANIFEST), + }, + "signature": "ef" * 32, + "algorithm": "hmac-sha256", + "schema_version": "prism_attestation_payload.v1", + "phase": "end", +} + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_PRISM_SRC = _REPO_ROOT / "packages" / "challenges" / "prism" / "src" + + +def ensure_prism_on_path() -> bool: + """Make prism_challenge importable without a separate uv env.""" + if not _PRISM_SRC.is_dir(): + return False + inserted = str(_PRISM_SRC) + if inserted not in sys.path: + sys.path.insert(0, inserted) + try: + import prism_challenge # noqa: F401 + + return True + except ImportError: + return False + + +def ok_record( + *, + work_unit_id: str = WORK_UNIT, + miner_hotkey: str = HOTKEY, + pod_id: str = POD, +) -> ConstationRunRecord: + return ConstationRunRecord( + ok=True, + reason=ConstationFailCode.OK, + fault_class=None, + miner_hotkey=miner_hotkey, + work_unit_id=work_unit_id, + pod_id=pod_id, + sidecar_digest=DIGEST, + lium_declared_digest=DIGEST, + constation_gap_budget_seconds=30.0, + constation_observed_max_gap_seconds=1.0, + corroboration_status=CorroborationStatus.AGREE, + samples=(), + ) + + +def fail_record( + *, + reason: ConstationFailCode = ConstationFailCode.REQUIRED_DIGEST_MISMATCH, +) -> ConstationRunRecord: + return ConstationRunRecord( + ok=False, + reason=reason, + fault_class=FaultClass.MINER, + miner_hotkey=HOTKEY, + work_unit_id=WORK_UNIT, + pod_id=POD, + sidecar_digest=None, + lium_declared_digest=None, + constation_gap_budget_seconds=30.0, + constation_observed_max_gap_seconds=0.0, + corroboration_status=CorroborationStatus.NOT_EVALUATED, + samples=(), + ) + + +@dataclass +class FakeRunner: + """Stand-in ConstationRunner for surface composition.""" + + poll_nonce_fn: Callable[[], str] | None = None + last_signed_wire: Mapping[str, Any] | None = field(default=None, init=False) + outcome: ConstationRunRecord = field(default_factory=ok_record) + wire: Mapping[str, Any] | None = field(default_factory=lambda: dict(WIRE)) + run_calls: int = 0 + last_request: ConstationRunRequest | None = None + phases_to_issue: tuple[str, ...] = ("start", "mid", "end") + + async def run(self, request: ConstationRunRequest) -> ConstationRunRecord: + self.run_calls += 1 + self.last_request = request + if self.outcome.ok and self.poll_nonce_fn is not None: + for _phase in self.phases_to_issue: + self.poll_nonce_fn() + self.last_signed_wire = dict(self.wire) if self.wire is not None else None + else: + self.last_signed_wire = None + if self.outcome.ok: + return ok_record( + work_unit_id=request.work_unit_id, + miner_hotkey=request.miner_hotkey, + pod_id=request.pod_id, + ) + return self.outcome + + +def binding_store(*, hotkey: str = HOTKEY, pod_id: str = POD) -> MinerPodBinding: + custody = LiumKeyCustody(master_key=generate_custody_master_key()) + custody.store_probed_key(miner_hotkey=hotkey, api_key="lium-surface-key-not-logged") + binding = MinerPodBinding(custody=custody) + binding._instance_by_hotkey[hotkey.strip()] = pod_id # noqa: SLF001 — test seam + return binding + + +def orchestration_request(**overrides: Any) -> ConstationOrchestrationRequest: + base: dict[str, Any] = { + "work_unit_id": WORK_UNIT, + "miner_hotkey": HOTKEY, + "required_digest": DIGEST, + "commit_sha": COMMIT, + "tree_sha": TREE, + "variant": ImageVariant.CUDA, + "sealed_manifest_hashes": dict(MANIFEST), + "duration_seconds": 5.0, + } + base.update(overrides) + return ConstationOrchestrationRequest(**base) + + +async def async_noop_sleep(_seconds: float) -> None: + return None + + +def make_orchestrator( + *, + pod_binding: MinerPodBinding | None = None, + nonce_service: AttestationNonceService | None = None, + bundle_store: ConstationBundleStore | None = None, + fake: FakeRunner | None = None, +) -> tuple[ + ProductionConstationOrchestrator, + AttestationNonceService, + ConstationBundleStore, + FakeRunner, +]: + binding = pod_binding if pod_binding is not None else binding_store() + nonces = ( + nonce_service + if nonce_service is not None + else AttestationNonceService(ttl=timedelta(hours=1), now_fn=lambda: T0) + ) + store = bundle_store if bundle_store is not None else ConstationBundleStore() + runner = fake if fake is not None else FakeRunner() + + def runner_factory(**kwargs: Any) -> FakeRunner: + runner.poll_nonce_fn = kwargs.get("poll_nonce_fn") + return runner + + orch = ProductionConstationOrchestrator( + pod_binding=binding, + nonce_service=nonces, + bundle_store=store, + poller_config=PollerConfig(), + now_fn=lambda: 0.0, + sleep_fn=async_noop_sleep, + rng_fn=lambda: 0.0, + runner_factory=runner_factory, + ) + return orch, nonces, store, runner + + +@contextmanager +def preserved_logging_levels() -> Iterator[None]: + """Restore logger levels after bittensor import side effects.""" + manager = logging.root.manager + prev_disable = manager.disable + prev_root_level = logging.getLogger().level + prev_levels = [ + (obj, obj.level, obj.disabled) + for obj in list(manager.loggerDict.values()) + if isinstance(obj, logging.Logger) + ] + try: + yield + finally: + for logger, level, disabled in prev_levels: + logger.setLevel(level) + logger.disabled = disabled + logging.getLogger().setLevel(prev_root_level) + manager.disable = prev_disable + + +class ChallengeRegistry: + async def get(self, slug: str) -> Any: + class R: + internal_base_url = "http://prism.surface.test" + + return R() + + async def get_token(self, slug: str) -> str: + return "tok" + + +class CaptureTransport(httpx.AsyncBaseTransport): + def __init__(self) -> None: + self.bodies: list[dict[str, Any]] = [] + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + import json + + self.bodies.append(json.loads(request.content.decode())) + return httpx.Response(200, json={"status": "accepted"}) + + +def minimal_proof() -> dict[str, Any]: + with preserved_logging_levels(): + import bittensor as bt + + from base.validator.agent.signing import KeypairRequestSigner + from base.worker.proof import build_execution_proof + + signer = KeypairRequestSigner(bt.Keypair.create_from_uri("//WorkerSurface")) + proof = build_execution_proof( + signer=signer, manifest_sha256="a" * 64, unit_id=WORK_UNIT + ) + return proof.model_dump(mode="json") diff --git a/tests/surface/test_constation_production_surface.py b/tests/surface/test_constation_production_surface.py new file mode 100644 index 00000000..0ee7f9c8 --- /dev/null +++ b/tests/surface/test_constation_production_surface.py @@ -0,0 +1,243 @@ +"""T12 surface core: S1/S2/S4/S5/S7 + B1s/B2s (composition, no live Lium).""" + +from __future__ import annotations + +import ast +import inspect +from typing import Any + +import pytest + +from base.attestation.payload import derive_attestation_key +from base.compute.attestation_nonce import ( + NonceBinding, + NonceConsumeHit, + NonceConsumeMiss, + NonceConsumeReason, +) +from base.compute.constation_triangle import evaluate_digest_triangle +from base.compute.constation_types import ConstationFailCode +from base.compute.digest_allowlist import DigestRecord, ImageVariant +from base.config.settings import ConstationSettings, Settings +from base.master.challenge_work_source import HttpChallengeResultForwarder +from base.master.constation.attestation_keys import load_attestation_verify_key +from base.master.constation.bundle_seal import seal_constation_bundle +from base.master.constation.orchestrator import ConstationOrchestrationResult +from tests.surface.constation_surface_helpers import ( + BUILD_SECRET, + COMMIT, + DIGEST, + DIGEST_BAD, + HOTKEY, + MANIFEST, + POD, + TREE, + WIRE, + WORK_UNIT, + CaptureTransport, + ChallengeRegistry, + FakeRunner, + fail_record, + make_orchestrator, + minimal_proof, + ok_record, + orchestration_request, +) + + +@pytest.mark.asyncio +async def test_s1_honest_path_seal_store_forward_nests_bundle() -> None: + """S1: orchestrator seals+puts; forwarder nests store bundle under result.""" + orch, _nonces, store, runner = make_orchestrator() + result = await orch.run(orchestration_request()) + + assert isinstance(result, ConstationOrchestrationResult) + assert result.ok is True + assert result.reason is ConstationFailCode.OK + assert runner.run_calls == 1 + stored = store.get(WORK_UNIT) + assert stored is not None + assert stored["work_unit_id"] == WORK_UNIT + assert stored["digest"] == DIGEST + assert stored["nonce"] == result.end_phase_nonce + assert "signed_attestation" in stored + assert stored["expected_sealed_manifest_hashes"] == dict(MANIFEST) + + allow = DigestRecord( + commit_sha=COMMIT, + tree_sha=TREE, + variant=ImageVariant.CUDA, + digest=DIGEST, + sealed_manifest_hashes=dict(MANIFEST), + ) + sealed = seal_constation_bundle( + allowlist_record=allow, + run_record=ok_record(), + nonce=result.end_phase_nonce or "n", + signed_attestation=dict(WIRE), + ) + assert sealed["digest"] == DIGEST + assert sealed["work_unit_id"] == WORK_UNIT + + transport = CaptureTransport() + + async def lookup(wu: str) -> dict[str, Any] | None: + return store.get(wu) + + fwd = HttpChallengeResultForwarder( + ChallengeRegistry(), transport=transport, retries=1, bundle_lookup=lookup + ) + await fwd.forward_result( + challenge_slug="prism", + work_unit_id=WORK_UNIT, + submission_ref=HOTKEY, + result_payload={"execution_proof": minimal_proof(), "executed": 1}, + ) + assert transport.bodies, "expected POST body" + assert transport.bodies[0]["result"]["constation_bundle"] == stored + + +@pytest.mark.asyncio +async def test_s2_adversarial_triangle_fail_no_ok_bundle_put() -> None: + """S2: triangle mismatch + runner fail → store stays empty.""" + tri = evaluate_digest_triangle( + required=DIGEST, + lium_declared=DIGEST, + sidecar=DIGEST_BAD, + ) + assert tri.ok is False + assert tri.fail_code is ConstationFailCode.REQUIRED_DIGEST_MISMATCH + + fake = FakeRunner( + outcome=fail_record(reason=ConstationFailCode.REQUIRED_DIGEST_MISMATCH) + ) + orch, _nonces, store, runner = make_orchestrator(fake=fake) + result = await orch.run(orchestration_request()) + + assert result.ok is False + assert result.reason is ConstationFailCode.REQUIRED_DIGEST_MISMATCH + assert store.get(WORK_UNIT) is None + assert result.bundle is None + assert runner.run_calls == 1 + + +@pytest.mark.asyncio +async def test_s4_nonce_issue_seal_consume_once_ok() -> None: + """S4/B2: after seal, end-phase nonce is first-consumable exactly once.""" + orch, nonces, store, _runner = make_orchestrator() + result = await orch.run(orchestration_request()) + + assert result.ok is True + assert result.end_phase_nonce is not None + stored = store.get(WORK_UNIT) + assert stored is not None + assert stored["nonce"] == result.end_phase_nonce + + binding = NonceBinding(work_unit_id=WORK_UNIT, miner_hotkey=HOTKEY, pod_id=POD) + first = nonces.consume(result.end_phase_nonce, binding) + assert isinstance(first, NonceConsumeHit), ( + f"end-phase nonce not first-consumable after seal (got {first!r})" + ) + second = nonces.consume(result.end_phase_nonce, binding) + assert isinstance(second, NonceConsumeMiss) + assert second.reason is NonceConsumeReason.ALREADY_CONSUMED + + +def test_s5_allowlist_sealed_hashes_required() -> None: + """S5: DigestRecord rejects empty sealed hashes; seal carries them.""" + with pytest.raises(ValueError, match="sealed_manifest_hashes must be non-empty"): + DigestRecord( + commit_sha=COMMIT, + tree_sha=TREE, + variant=ImageVariant.CUDA, + digest=DIGEST, + sealed_manifest_hashes={}, + ) + + allow = DigestRecord( + commit_sha=COMMIT, + tree_sha=TREE, + variant=ImageVariant.CUDA, + digest=DIGEST, + sealed_manifest_hashes=dict(MANIFEST), + ) + out = seal_constation_bundle( + allowlist_record=allow, + run_record=ok_record(), + nonce="surface-nonce", + signed_attestation=dict(WIRE), + ) + assert out["expected_sealed_manifest_hashes"] == dict(MANIFEST) + assert out["reported_sealed_manifest_hashes"] == dict(MANIFEST) + + +@pytest.mark.asyncio +async def test_s7_forwarder_embed_from_lookup() -> None: + """S7: forwarder embeds result.constation_bundle from lookup.""" + transport = CaptureTransport() + bundle = { + "digest": DIGEST, + "nonce": "n-surface", + "work_unit_id": WORK_UNIT, + } + + async def lookup(wu: str) -> dict[str, Any] | None: + return bundle if wu == WORK_UNIT else None + + fwd = HttpChallengeResultForwarder( + ChallengeRegistry(), transport=transport, retries=1, bundle_lookup=lookup + ) + await fwd.forward_result( + challenge_slug="prism", + work_unit_id=WORK_UNIT, + submission_ref=HOTKEY, + result_payload={"execution_proof": minimal_proof(), "executed": 1}, + ) + assert transport.bodies + assert transport.bodies[0]["result"]["constation_bundle"] == bundle + + +def test_b1s_verify_key_wiring_smoke() -> None: + """B1s: load_attestation_verify_key + main AST wires key kwarg.""" + key = derive_attestation_key(BUILD_SECRET) + settings = Settings( + constation=ConstationSettings(attestation_verify_key_hex=key.hex()) + ) + assert load_attestation_verify_key(settings) == key + assert load_attestation_verify_key(Settings()) is None + + import base.cli_app.main as main_mod + + source = inspect.getsource(main_mod) + tree = ast.parse(source) + hits: list[ast.Call] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + name = ( + func.id + if isinstance(func, ast.Name) + else func.attr + if isinstance(func, ast.Attribute) + else None + ) + if name != "build_constation_router": + continue + hits.append(node) + assert hits, "build_constation_router call missing from main" + for call in hits: + kw_names = {kw.arg for kw in call.keywords if kw.arg is not None} + assert "attestation_verify_key" in kw_names, ( + "build_constation_router must receive attestation_verify_key= " + f"(got keywords {sorted(kw_names)})" + ) + + +def test_b2s_orchestrator_source_never_calls_consume() -> None: + """B2s: orchestrator module must not call nonce.consume.""" + from base.master.constation import orchestrator as orch_mod + + src = inspect.getsource(orch_mod) + assert ".consume(" not in src + assert "nonce_service.consume" not in src diff --git a/tests/surface/test_constation_production_surface_prism_http.py b/tests/surface/test_constation_production_surface_prism_http.py new file mode 100644 index 00000000..d1420eef --- /dev/null +++ b/tests/surface/test_constation_production_surface_prism_http.py @@ -0,0 +1,158 @@ +"""T12 surface prism + HTTP: S3 missing bundle, S6 legacy gate, S8 challenge route.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from datetime import timedelta +from pathlib import Path +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from base.attestation.payload import derive_attestation_key +from base.compute.attestation_nonce import NonceBinding +from base.db.models import Base +from base.master.constation.allowlist_repository import DigestAllowlistRepository +from base.master.constation.bundle_store import ConstationBundleStore +from base.master.constation.nonce_repository import DurableAttestationNonceService +from base.master.constation.routes import create_constation_test_app +from tests.surface.constation_surface_helpers import ( + BUILD_SECRET, + DIGEST, + HOTKEY, + POD, + TOKEN, + WORK_UNIT, + ensure_prism_on_path, +) + + +def test_s3_missing_bundle_prism_behavior() -> None: + """S3: prism rejects elevation / ingest kwargs without constation_bundle.""" + if not ensure_prism_on_path(): + pytest.skip("prism_challenge not importable (packages/challenges/prism/src)") + + from prism_challenge.app import _constation_ingest_kwargs + from prism_challenge.audit import effective_tier + from prism_challenge.config import PrismSettings + from prism_challenge.ingestion import miner_fault_reason + from prism_challenge.proof import ExecutionProof, ProviderInfo, WorkerSignature + + settings = PrismSettings( + allow_insecure_signatures=False, + constation_base_url="http://base.surface.test", + constation_internal_token="tok", + ) + assert _constation_ingest_kwargs(settings, {"executed": 1}) == {} + + proof = ExecutionProof( + version=1, + tier=1, + manifest_sha256="c" * 64, + image_digest=DIGEST, + provider=ProviderInfo(name="lium", pod_id=POD), + worker_signature=WorkerSignature(worker_pubkey="wk", sig="0xab"), + ) + assert effective_tier(proof, pinned_image_digest=DIGEST) == 0 + assert ( + effective_tier(proof, pinned_image_digest=DIGEST, constation_ok_result=False) + == 0 + ) + assert miner_fault_reason("missing_constation_bundle") == ( + "miner_fault:missing_constation_bundle" + ) + + +def test_s6_legacy_gate_no_elevation_without_constation() -> None: + """S6: self-report digest match alone cannot elevate (tier stays 0).""" + if not ensure_prism_on_path(): + pytest.skip("prism_challenge not importable (packages/challenges/prism/src)") + + from prism_challenge.audit import effective_tier + from prism_challenge.proof import ExecutionProof, ProviderInfo, WorkerSignature + + proof = ExecutionProof( + version=1, + tier=1, + manifest_sha256="c" * 64, + image_digest=DIGEST, + provider=ProviderInfo(name="lium", pod_id=POD), + worker_signature=WorkerSignature(worker_pubkey="wk", sig="0xab"), + ) + assert effective_tier(proof, pinned_image_digest=DIGEST) == 0 + assert ( + effective_tier(proof, pinned_image_digest=DIGEST, constation_ok_result=None) + == 0 + ) + assert ( + effective_tier(proof, pinned_image_digest=DIGEST, constation_ok_result=True) + == 1 + ) + + +@pytest.fixture +async def challenge_harness(tmp_path: Path) -> AsyncIterator[dict[str, Any]]: + db_path = tmp_path / "surface_challenge.sqlite3" + engine = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", + connect_args={"check_same_thread": False}, + ) + factory = async_sessionmaker(engine, expire_on_commit=False, autoflush=False) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + binding = NonceBinding(work_unit_id=WORK_UNIT, miner_hotkey=HOTKEY, pod_id=POD) + key = derive_attestation_key(BUILD_SECRET) + app = create_constation_test_app( + allowlist_repo=DigestAllowlistRepository(factory), + nonce_service=DurableAttestationNonceService(factory, ttl=timedelta(hours=1)), + internal_token=TOKEN, + default_binding=binding, + attestation_verify_key=key, + bundle_store=ConstationBundleStore(), + ) + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + yield { + "client": client, + "headers": {"Authorization": f"Bearer {TOKEN}"}, + "key": key, + } + await engine.dispose() + + +@pytest.mark.asyncio +async def test_s8_challenge_route_smoke(challenge_harness: dict[str, Any]) -> None: + """S8: GET challenge + POST answer + issue_nonce smoke.""" + client: AsyncClient = challenge_harness["client"] + headers: dict[str, str] = challenge_harness["headers"] + + r = await client.get("/v1/attestation/challenge", params={"phase": "start"}) + assert r.status_code == 200, r.text + body = r.json() + assert body["nonce"] + assert body["phase"] == "start" + assert body["work_unit_id"] == WORK_UNIT + + ans = await client.post( + "/v1/attestation/answer", + json={"nonce": body["nonce"], "phase": "start"}, + ) + assert ans.status_code == 200 + assert ans.json()["status"] == "accepted" + + issued = await client.post( + "/internal/v1/constation/issue_nonce", + headers=headers, + json={ + "work_unit_id": WORK_UNIT, + "miner_hotkey": HOTKEY, + "pod_id": POD, + "phase": "end", + }, + ) + assert issued.status_code == 200, issued.text + assert issued.json()["nonce"] + assert issued.json()["phase"] == "end" diff --git a/tests/unit/test_attestation_http.py b/tests/unit/test_attestation_http.py index a6b63316..2ed88b16 100644 --- a/tests/unit/test_attestation_http.py +++ b/tests/unit/test_attestation_http.py @@ -96,6 +96,7 @@ async def test_register_and_check_allowlist(harness: dict[str, Any]) -> None: "tree_sha": TREE, "variant": "cuda", "digest": DIGEST, + "sealed_manifest_hashes": dict(MANIFEST), }, ) assert reg.status_code == 200, reg.text @@ -214,3 +215,305 @@ async def test_verify_attestation_ok(harness: dict[str, Any]) -> None: ) assert r.status_code == 200, r.text assert r.json()["ok"] is True + + +@pytest.mark.asyncio +async def test_register_digest_http_rejects_empty_sealed_manifest( + harness: dict[str, Any], +) -> None: + """Given empty sealed hashes, When POST register_digest, Then 4xx not 500.""" + client = harness["client"] + headers = harness["headers"] + reg = await client.post( + "/internal/v1/constation/register_digest", + headers=headers, + json={ + "commit_sha": COMMIT, + "tree_sha": TREE, + "variant": "cuda", + "digest": DIGEST, + "sealed_manifest_hashes": {}, + }, + ) + assert 400 <= reg.status_code < 500, reg.text + assert reg.status_code != 500 + + +# --------------------------------------------------------------------------- +# T6b register_miner_key +# --------------------------------------------------------------------------- + + +class _FakePodBinding: + """Minimal stand-in for MinerPodBinding.register (HTTP layer only).""" + + def __init__( + self, + *, + verdict: object | None = None, + raise_value_error: str | None = None, + ) -> None: + from base.compute.constation_types import ConstationFailCode, ConstationVerdict + + self.calls: list[dict[str, str]] = [] + self._raise = raise_value_error + self._verdict = ( + verdict + if verdict is not None + else ConstationVerdict(ok=True, reason=ConstationFailCode.OK) + ) + + async def register( + self, + *, + miner_hotkey: str, + api_key: str, + instance_id: str, + ) -> object: + self.calls.append( + { + "miner_hotkey": miner_hotkey, + "api_key": api_key, + "instance_id": instance_id, + } + ) + if self._raise is not None: + raise ValueError(self._raise) + return self._verdict + + +@pytest.fixture +async def register_miner_harness( + tmp_path: Path, +) -> AsyncIterator[dict[str, Any]]: + from fastapi import FastAPI + + from base.master.constation.routes import build_constation_router + + db_path = tmp_path / "register_miner_http.sqlite3" + engine = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", + connect_args={"check_same_thread": False}, + ) + factory = async_sessionmaker(engine, expire_on_commit=False, autoflush=False) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + allowlist_repo = DigestAllowlistRepository(factory) + nonce_svc = DurableAttestationNonceService(factory, ttl=timedelta(hours=1)) + fake = _FakePodBinding() + app = FastAPI() + router = build_constation_router( + allowlist_repo=allowlist_repo, + nonce_service=nonce_svc, + internal_token=TOKEN, + pod_binding=fake, # type: ignore[arg-type] + ) + app.include_router(router) + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + yield { + "client": client, + "headers": {"Authorization": f"Bearer {TOKEN}"}, + "binding": fake, + "app": app, + "allowlist_repo": allowlist_repo, + "nonce_svc": nonce_svc, + } + await engine.dispose() + + +@pytest.mark.asyncio +async def test_register_miner_key_ok( + register_miner_harness: dict[str, Any], +) -> None: + """Given valid body + binding ok, When POST, Then 200 registered.""" + client: AsyncClient = register_miner_harness["client"] + headers = register_miner_harness["headers"] + binding: _FakePodBinding = register_miner_harness["binding"] + secret = "lium-http-test-key-NEVER-ECHO" + + r = await client.post( + "/internal/v1/constation/register_miner_key", + headers=headers, + json={ + "miner_hotkey": "hk-miner-1", + "api_key": secret, + "instance_id": "pod-xyz", + }, + ) + assert r.status_code == 200, r.text + body = r.json() + assert body == {"status": "registered"} + assert "api_key" not in body + assert secret not in r.text + assert binding.calls == [ + { + "miner_hotkey": "hk-miner-1", + "api_key": secret, + "instance_id": "pod-xyz", + } + ] + + +@pytest.mark.asyncio +async def test_register_miner_key_verdict_fail_is_422( + tmp_path: Path, +) -> None: + """Given fail verdict, When POST, Then 422 with fail code not 500.""" + from fastapi import FastAPI + + from base.compute.constation_types import ConstationFailCode, ConstationVerdict + from base.master.constation.routes import build_constation_router + + db_path = tmp_path / "register_miner_fail.sqlite3" + engine = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", + connect_args={"check_same_thread": False}, + ) + factory = async_sessionmaker(engine, expire_on_commit=False, autoflush=False) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + fake = _FakePodBinding( + verdict=ConstationVerdict( + ok=False, + reason=ConstationFailCode.LIUM_AUTH_REVOKED, + detail="probe_401", + ) + ) + app = FastAPI() + app.include_router( + build_constation_router( + allowlist_repo=DigestAllowlistRepository(factory), + nonce_service=DurableAttestationNonceService( + factory, ttl=timedelta(hours=1) + ), + internal_token=TOKEN, + pod_binding=fake, # type: ignore[arg-type] + ) + ) + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + r = await client.post( + "/internal/v1/constation/register_miner_key", + headers={"Authorization": f"Bearer {TOKEN}"}, + json={ + "miner_hotkey": "hk-1", + "api_key": "bad-key-secret", + "instance_id": "pod-1", + }, + ) + await engine.dispose() + assert r.status_code == 422, r.text + assert r.status_code != 500 + detail = r.json()["detail"] + assert "lium_auth_revoked" in str(detail) + assert "bad-key-secret" not in r.text + + +@pytest.mark.asyncio +async def test_register_miner_key_without_binding_is_503( + tmp_path: Path, +) -> None: + """Given pod_binding=None, When POST register_miner_key, Then 503.""" + from fastapi import FastAPI + + from base.master.constation.routes import build_constation_router + + db_path = tmp_path / "register_miner_503.sqlite3" + engine = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", + connect_args={"check_same_thread": False}, + ) + factory = async_sessionmaker(engine, expire_on_commit=False, autoflush=False) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + app = FastAPI() + app.include_router( + build_constation_router( + allowlist_repo=DigestAllowlistRepository(factory), + nonce_service=DurableAttestationNonceService( + factory, ttl=timedelta(hours=1) + ), + internal_token=TOKEN, + pod_binding=None, + ) + ) + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + r = await client.post( + "/internal/v1/constation/register_miner_key", + headers={"Authorization": f"Bearer {TOKEN}"}, + json={ + "miner_hotkey": "hk-1", + "api_key": "k", + "instance_id": "pod-1", + }, + ) + await engine.dispose() + assert r.status_code == 503, r.text + + +@pytest.mark.asyncio +async def test_register_miner_key_requires_internal_auth( + register_miner_harness: dict[str, Any], +) -> None: + """Given no bearer, When POST register_miner_key, Then 401.""" + client: AsyncClient = register_miner_harness["client"] + r = await client.post( + "/internal/v1/constation/register_miner_key", + json={ + "miner_hotkey": "hk-1", + "api_key": "k", + "instance_id": "pod-1", + }, + ) + assert r.status_code == 401 + + +@pytest.mark.asyncio +async def test_register_miner_key_value_error_is_422( + tmp_path: Path, +) -> None: + """Given binding raises ValueError, When POST, Then 422 not 500.""" + from fastapi import FastAPI + + from base.master.constation.routes import build_constation_router + + db_path = tmp_path / "register_miner_ve.sqlite3" + engine = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", + connect_args={"check_same_thread": False}, + ) + factory = async_sessionmaker(engine, expire_on_commit=False, autoflush=False) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + fake = _FakePodBinding(raise_value_error="miner_hotkey must be a non-empty string") + app = FastAPI() + app.include_router( + build_constation_router( + allowlist_repo=DigestAllowlistRepository(factory), + nonce_service=DurableAttestationNonceService( + factory, ttl=timedelta(hours=1) + ), + internal_token=TOKEN, + pod_binding=fake, # type: ignore[arg-type] + ) + ) + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + r = await client.post( + "/internal/v1/constation/register_miner_key", + headers={"Authorization": f"Bearer {TOKEN}"}, + json={ + "miner_hotkey": " ", + "api_key": "k", + "instance_id": "pod-1", + }, + ) + await engine.dispose() + assert r.status_code == 422, r.text + assert r.status_code != 500 diff --git a/tests/unit/test_attestation_verify_key_wiring.py b/tests/unit/test_attestation_verify_key_wiring.py new file mode 100644 index 00000000..651fd4e4 --- /dev/null +++ b/tests/unit/test_attestation_verify_key_wiring.py @@ -0,0 +1,228 @@ +"""B1: master must load attestation_verify_key_hex and pass it to the router. + +Prism BaseHttp verify_signature sends no body key_hex — production relies on +the router-held key. If main omits attestation_verify_key=, verify always +returns empty_key (the pre-fix bug). +""" + +from __future__ import annotations + +import ast +import inspect +from collections.abc import AsyncIterator +from datetime import timedelta +from pathlib import Path +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from base.attestation.payload import ( + AttestationPayload, + compute_build_secret_response, + derive_attestation_key, + sign_attestation_payload, +) +from base.compute.attestation_nonce import NonceBinding +from base.config.settings import ConstationSettings, Settings +from base.db.models import Base +from base.master.constation.allowlist_repository import DigestAllowlistRepository +from base.master.constation.attestation_keys import load_attestation_verify_key +from base.master.constation.nonce_repository import DurableAttestationNonceService +from base.master.constation.routes import create_constation_test_app + +TOKEN = "test-internal" +BINDING = NonceBinding(work_unit_id="wu-wire", miner_hotkey="hk-1", pod_id="pod-1") +DIGEST = "sha256:" + ("c" * 64) +MANIFEST = {"harness.py": "d" * 64} +BUILD_SECRET = b"build-secret-fixture" + + +def test_load_attestation_verify_key_missing_returns_none() -> None: + """Given no hex setting, When load, Then None (fail-closed → empty_key).""" + assert load_attestation_verify_key(Settings()) is None + assert ( + load_attestation_verify_key(Settings(constation=ConstationSettings())) is None + ) + assert ( + load_attestation_verify_key( + Settings(constation=ConstationSettings(attestation_verify_key_hex="")) + ) + is None + ) + assert ( + load_attestation_verify_key( + Settings(constation=ConstationSettings(attestation_verify_key_hex=" ")) + ) + is None + ) + + +def test_load_attestation_verify_key_hex_roundtrip() -> None: + """Given hex key in settings, When load, Then raw key bytes.""" + key = derive_attestation_key(BUILD_SECRET) + settings = Settings( + constation=ConstationSettings(attestation_verify_key_hex=key.hex()) + ) + assert load_attestation_verify_key(settings) == key + + +def test_main_passes_attestation_verify_key_into_build_constation_router() -> None: + """Given main.py call site, When AST-inspected, Then key kwarg is wired. + + Drops of ``attestation_verify_key=...`` at the production call site must + fail this test (regression guard for the empty_key production bug). + """ + import base.cli_app.main as main_mod + + source = inspect.getsource(main_mod) + tree = ast.parse(source) + hits: list[ast.Call] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + name = ( + func.id + if isinstance(func, ast.Name) + else func.attr + if isinstance(func, ast.Attribute) + else None + ) + if name != "build_constation_router": + continue + hits.append(node) + assert hits, "build_constation_router call missing from main" + for call in hits: + kw_names = {kw.arg for kw in call.keywords if kw.arg is not None} + assert "attestation_verify_key" in kw_names, ( + "build_constation_router must receive attestation_verify_key= " + f"(got keywords {sorted(kw_names)})" + ) + + +@pytest.fixture +async def wired_harness(tmp_path: Path) -> AsyncIterator[dict[str, Any]]: + """Router built the production way: key from settings, not body key_hex.""" + key = derive_attestation_key(BUILD_SECRET) + settings = Settings( + constation=ConstationSettings(attestation_verify_key_hex=key.hex()) + ) + loaded = load_attestation_verify_key(settings) + assert loaded == key + + db_path = tmp_path / "constation_wire.sqlite3" + engine = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", + connect_args={"check_same_thread": False}, + ) + factory = async_sessionmaker(engine, expire_on_commit=False, autoflush=False) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + app = create_constation_test_app( + allowlist_repo=DigestAllowlistRepository(factory), + nonce_service=DurableAttestationNonceService(factory, ttl=timedelta(hours=1)), + internal_token=TOKEN, + default_binding=BINDING, + attestation_verify_key=loaded, + ) + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + yield { + "client": client, + "headers": {"Authorization": f"Bearer {TOKEN}"}, + "key": key, + } + await engine.dispose() + + +@pytest.fixture +async def unwired_harness(tmp_path: Path) -> AsyncIterator[dict[str, Any]]: + """Old bug: router built without attestation_verify_key.""" + db_path = tmp_path / "constation_unwire.sqlite3" + engine = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", + connect_args={"check_same_thread": False}, + ) + factory = async_sessionmaker(engine, expire_on_commit=False, autoflush=False) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + app = create_constation_test_app( + allowlist_repo=DigestAllowlistRepository(factory), + nonce_service=DurableAttestationNonceService(factory, ttl=timedelta(hours=1)), + internal_token=TOKEN, + default_binding=BINDING, + # intentionally omit attestation_verify_key + ) + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + yield {"client": client, "headers": {"Authorization": f"Bearer {TOKEN}"}} + await engine.dispose() + + +def _signed_wire(*, key: bytes) -> dict[str, Any]: + payload = AttestationPayload( + nonce="nonce-wire-1", + digest=DIGEST, + pod_id=BINDING.pod_id, + variant="cuda", + sealed_manifest_hashes=dict(MANIFEST), + build_secret_response=compute_build_secret_response( + build_secret=BUILD_SECRET, nonce="nonce-wire-1" + ), + ) + signed = sign_attestation_payload(payload, signing_key=key) + return { + "payload": { + "nonce": payload.nonce, + "digest": payload.digest, + "pod_id": payload.pod_id, + "variant": payload.variant, + "sealed_manifest_hashes": dict(payload.sealed_manifest_hashes), + "build_secret_response": payload.build_secret_response, + }, + "signature": signed.signature, + "algorithm": signed.algorithm, + "schema_version": signed.schema_version, + } + + +@pytest.mark.asyncio +async def test_verify_attestation_ok_without_body_key_hex_when_router_wired( + wired_harness: dict[str, Any], +) -> None: + """Given settings hex key on router, When verify without key_hex, Then ok.""" + client: AsyncClient = wired_harness["client"] + headers = wired_harness["headers"] + key: bytes = wired_harness["key"] + r = await client.post( + "/internal/v1/constation/verify_attestation", + headers=headers, + json={"signed": _signed_wire(key=key)}, + ) + assert r.status_code == 200, r.text + body = r.json() + assert body["ok"] is True + assert body["reason"] == "ok" + + +@pytest.mark.asyncio +async def test_verify_attestation_empty_key_when_router_unwired( + unwired_harness: dict[str, Any], +) -> None: + """Given router without key (old bug), When verify sans key_hex, Then empty_key.""" + client: AsyncClient = unwired_harness["client"] + headers = unwired_harness["headers"] + key = derive_attestation_key(BUILD_SECRET) + r = await client.post( + "/internal/v1/constation/verify_attestation", + headers=headers, + json={"signed": _signed_wire(key=key)}, + ) + assert r.status_code == 200, r.text + body = r.json() + assert body["ok"] is False + assert body["reason"] == "empty_key" diff --git a/tests/unit/test_compute_constation_service.py b/tests/unit/test_compute_constation_service.py index ce5866dd..13a5635e 100644 --- a/tests/unit/test_compute_constation_service.py +++ b/tests/unit/test_compute_constation_service.py @@ -93,6 +93,8 @@ class ScriptedLium: """Minimal LiumClient stand-in for runner unit tests.""" digests: list[str | None] = field(default_factory=lambda: [DIGEST_A]) + miner_hotkey: str = HOTKEY + status: str = "RUNNING" auth_fail_on_call: int | None = None rate_limit_on_call: int | None = None network_fail_times: int = 0 @@ -120,13 +122,34 @@ async def get_pod_raw(self, pod_id: str) -> LiumPodRead: pod_id=pod_id, template_id="tmpl-1", docker_image_digest=digest, - raw={"id": pod_id, "template": {"docker_image_digest": digest}}, + raw={ + "id": pod_id, + "status": self.status, + "executor": { + "miner_hotkey": self.miner_hotkey, + "executor_ip_address": "10.0.0.1", + }, + "template": {"docker_image_digest": digest}, + }, ) async def balance(self) -> float: return 1.0 +def _run_request(**overrides: object) -> ConstationRunRequest: + """Build a ConstationRunRequest with triangle-ready defaults.""" + payload: dict[str, object] = { + "miner_hotkey": HOTKEY, + "work_unit_id": WORK_UNIT, + "pod_id": POD, + "duration_seconds": 0.0, + "required_digest": DIGEST_A, + } + payload.update(overrides) + return ConstationRunRequest(**payload) # type: ignore[arg-type] + + def _custody( *, factory: Any | None = None, @@ -235,6 +258,7 @@ def factory(key: str) -> Any: work_unit_id=WORK_UNIT, pod_id=POD, duration_seconds=10.0, + required_digest=DIGEST_A, ) ) assert record.ok is False @@ -506,6 +530,7 @@ async def _probe(client: Any) -> None: work_unit_id=WORK_UNIT, pod_id=POD, duration_seconds=10.0, + required_digest=DIGEST_A, ) ) assert record.ok is True @@ -557,12 +582,14 @@ async def _probe(client: Any) -> None: work_unit_id=WORK_UNIT, pod_id=POD, duration_seconds=0.0, + required_digest=DIGEST_A, ) ) assert record.ok is False - assert record.reason is ConstationFailCode.CORROBORATION_MISMATCH + # Triangle: sidecar != required takes precedence over two-way corroboration. + assert record.reason is ConstationFailCode.REQUIRED_DIGEST_MISMATCH assert record.fault_class is FaultClass.MINER - assert record.corroboration_status is CorroborationStatus.MISMATCH + assert record.corroboration_status is CorroborationStatus.NOT_EVALUATED def test_corroboration_agree_insufficient_for_elevation_contract() -> None: @@ -599,7 +626,234 @@ async def test_runner_unregistered_key_fail_closed() -> None: work_unit_id=WORK_UNIT, pod_id=POD, duration_seconds=0.0, + required_digest=DIGEST_A, ) ) assert record.ok is False assert record.reason is ConstationFailCode.KEY_NOT_REGISTERED + + +# =========================================================================== +# T5 — runner poll: pod bind + digest triangle +# =========================================================================== + + +async def _registered_runner( + *, + scripted: ScriptedLium, + sidecar: FakeSidecar | None = None, + duration_seconds: float = 0.0, + required_digest: str = DIGEST_A, + attestor_factory: object | None = None, +) -> tuple[ConstationRunner, ConstationRunRequest]: + def factory(key: str) -> object: + del key + return scripted + + async def _probe(client: object) -> None: + del client + await scripted.balance() + + custody = _custody(factory=factory) + custody.probe_fn = _probe + await custody.register(miner_hotkey=HOTKEY, api_key=API_KEY) + clock = FakeClock() + kwargs: dict[str, object] = {} + if attestor_factory is not None: + kwargs["attestor_factory"] = attestor_factory + runner = ConstationRunner( + custody=custody, + sidecar=sidecar or FakeSidecar(digest=DIGEST_A), + poller_config=PollerConfig( + gap_budget_seconds=60.0, + min_interval_seconds=5.0, + max_interval_seconds=5.0, + max_polls=10, + max_cost_units=10.0, + rate_limit_per_second=100.0, + ), + now_fn=clock.now, + sleep_fn=clock.sleep, + rng_fn=SequenceRng([0.0]), + **kwargs, # type: ignore[arg-type] + ) + req = _run_request( + duration_seconds=duration_seconds, + required_digest=required_digest, + ) + return runner, req + + +async def test_runner_triangle_happy_path_agree() -> None: + """S1: RUNNING + hotkey bind + required==lium==sidecar → ok AGREE.""" + runner, req = await _registered_runner( + scripted=ScriptedLium(digests=[DIGEST_A]), + sidecar=FakeSidecar(digest=DIGEST_A), + duration_seconds=10.0, + ) + record = await runner.run(req) + assert record.ok is True + assert record.reason is ConstationFailCode.OK + assert record.corroboration_status is CorroborationStatus.AGREE + assert record.sidecar_digest == DIGEST_A + assert record.lium_declared_digest == DIGEST_A + assert len(record.samples) >= 2 + + +async def test_runner_triangle_sidecar_required_mismatch() -> None: + """S2: sidecar actual != required → REQUIRED_DIGEST_MISMATCH (triangle first).""" + runner, req = await _registered_runner( + scripted=ScriptedLium(digests=[DIGEST_A]), + sidecar=FakeSidecar(digest=DIGEST_B), + required_digest=DIGEST_A, + ) + record = await runner.run(req) + assert record.ok is False + assert record.reason is ConstationFailCode.REQUIRED_DIGEST_MISMATCH + assert record.fault_class is FaultClass.MINER + + +async def test_runner_triangle_lium_mismatch_when_sidecar_matches_required() -> None: + """Triple path: sidecar==required but lium differs → CORROBORATION_MISMATCH.""" + runner, req = await _registered_runner( + scripted=ScriptedLium(digests=[DIGEST_B]), + sidecar=FakeSidecar(digest=DIGEST_A), + required_digest=DIGEST_A, + ) + record = await runner.run(req) + assert record.ok is False + assert record.reason is ConstationFailCode.CORROBORATION_MISMATCH + assert record.fault_class is FaultClass.MINER + assert record.corroboration_status is CorroborationStatus.MISMATCH + + +async def test_runner_triangle_absent_lium_digest_fail_closed() -> None: + """S3: Lium declared digest absent → LIUM_DIGEST_ABSENT (no longer optional-ok).""" + runner, req = await _registered_runner( + scripted=ScriptedLium(digests=[None]), + sidecar=FakeSidecar(digest=DIGEST_A), + required_digest=DIGEST_A, + ) + record = await runner.run(req) + assert record.ok is False + assert record.reason is ConstationFailCode.LIUM_DIGEST_ABSENT + assert record.fault_class is FaultClass.MINER + + +async def test_runner_pod_hotkey_mismatch_fail_closed() -> None: + """S4: executor.miner_hotkey != request.miner_hotkey → POD_HOTKEY_MISMATCH.""" + runner, req = await _registered_runner( + scripted=ScriptedLium(digests=[DIGEST_A], miner_hotkey="5OtherMinerHotkeyXXXX"), + sidecar=FakeSidecar(digest=DIGEST_A), + ) + record = await runner.run(req) + assert record.ok is False + assert record.reason is ConstationFailCode.POD_HOTKEY_MISMATCH + assert record.fault_class is FaultClass.MINER + + +async def test_runner_pod_not_running_fail_closed() -> None: + """S5: status not RUNNING → POD_NOT_RUNNING.""" + runner, req = await _registered_runner( + scripted=ScriptedLium(digests=[DIGEST_A], status="STOPPED"), + sidecar=FakeSidecar(digest=DIGEST_A), + ) + record = await runner.run(req) + assert record.ok is False + assert record.reason is ConstationFailCode.POD_NOT_RUNNING + assert record.fault_class is FaultClass.MINER + + +async def test_runner_attestor_factory_re_resolves_each_poll() -> None: + """Factory is invoked per poll so mid-run port remap can re-bind.""" + scripted = ScriptedLium(digests=[DIGEST_A]) + seen: list[object] = [] + + def factory(pod_raw: object) -> FakeSidecar: + seen.append(pod_raw) + return FakeSidecar(digest=DIGEST_A) + + runner, req = await _registered_runner( + scripted=scripted, + duration_seconds=10.0, + attestor_factory=factory, + ) + record = await runner.run(req) + assert record.ok is True + assert len(seen) >= 2 + + +async def test_runner_retains_last_signed_wire_from_http_hit() -> None: + """When attestor returns a signed wire (Http path), runner keeps last copy.""" + from base.compute.constation_sidecar_client import SidecarAttestHit + + wire = { + "payload": { + "digest": DIGEST_A, + "nonce": "n1", + "pod_id": POD, + "sealed_manifest_hashes": {"a": "b"}, + }, + "signature": "sig", + "algorithm": "hmac-sha256", + "schema_version": "prism_attestation_payload.v1", + "phase": "start", + } + + @dataclass + class WireAttestor: + async def attest(self, *, nonce: str, phase: str) -> SidecarAttestHit: + del nonce + return SidecarAttestHit( + digest=DIGEST_A, + nonce="n1", + pod_id=POD, + phase=phase, + signature="sig", + algorithm="hmac-sha256", + schema_version="prism_attestation_payload.v1", + sealed_manifest_hashes={"a": "b"}, + wire=wire, + ) + + # Wrap as HttpSidecarAttestor-shaped via factory returning object with Hit path: + # Use a thin adapter that runner detects only for HttpSidecarAttestor. + # Instead, monkey via factory returning Protocol str attestor is insufficient. + # Directly set last_signed_wire through a custom path: subclass Http check. + # Prefer factory returning an object the runner treats as Protocol (str) — + # wire retention requires HttpSidecarAttestor. Build a stub subclass. + + from base.compute.constation_sidecar_client import HttpSidecarAttestor + + class StubHttp(HttpSidecarAttestor): + def __init__(self) -> None: + # bypass __post_init__ URL validation via object.__new__ + object.__setattr__(self, "base_url", "http://10.0.0.1:9") + object.__setattr__(self, "timeout_seconds", 1.0) + object.__setattr__(self, "transport", None) + + async def attest(self, *, nonce: str, phase: str) -> SidecarAttestHit: + del nonce + return SidecarAttestHit( + digest=DIGEST_A, + nonce="n1", + pod_id=POD, + phase=phase, + signature="sig", + algorithm="hmac-sha256", + schema_version="prism_attestation_payload.v1", + sealed_manifest_hashes={"a": "b"}, + wire=wire, + ) + + def factory(pod_raw: object) -> StubHttp: + del pod_raw + return StubHttp() + + runner, req = await _registered_runner( + scripted=ScriptedLium(digests=[DIGEST_A]), + attestor_factory=factory, + ) + record = await runner.run(req) + assert record.ok is True + assert runner.last_signed_wire == wire diff --git a/tests/unit/test_compute_digest_allowlist.py b/tests/unit/test_compute_digest_allowlist.py index 7d646e5e..1ada727e 100644 --- a/tests/unit/test_compute_digest_allowlist.py +++ b/tests/unit/test_compute_digest_allowlist.py @@ -36,12 +36,19 @@ def _record( tree_sha: str = TREE_A, variant: ImageVariant = ImageVariant.CUDA, digest: str = DIGEST_CUDA, + sealed_manifest_hashes: dict[str, str] | None = None, ) -> DigestRecord: + hashes = ( + sealed_manifest_hashes + if sealed_manifest_hashes is not None + else {"default.py": "e" * 64} + ) return DigestRecord( commit_sha=commit_sha, tree_sha=tree_sha, variant=variant, digest=digest, + sealed_manifest_hashes=hashes, ) @@ -295,3 +302,65 @@ def test_alembic_migration_0017_is_chained_from_watcher_state() -> None: assert "image_digest_allowlist" in text assert "denied_image_digests" in text assert "denied_image_commits" in text + + +SEALED_HASHES = {"harness.py": "a" * 64, "runner.py": "b" * 64} + + +def test_register_digest_rejects_empty_sealed_manifest() -> None: + """Given empty sealed hashes, When building DigestRecord, Then ValueError.""" + with pytest.raises(ValueError, match="sealed_manifest_hashes"): + DigestRecord( + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + digest=DIGEST_CUDA, + sealed_manifest_hashes={}, + ) + + +def test_register_digest_rejects_non_hex_hash_value() -> None: + """Given non-64-hex hash, When building DigestRecord, Then ValueError.""" + with pytest.raises(ValueError, match="sealed_manifest_hashes"): + DigestRecord( + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + digest=DIGEST_CUDA, + sealed_manifest_hashes={"harness.py": "not-a-hex-digest"}, + ) + + +def test_register_digest_rejects_blank_path_key() -> None: + """Given a blank path key, When constructing DigestRecord, Then ValueError.""" + with pytest.raises(ValueError, match="sealed_manifest_hashes"): + DigestRecord( + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + digest=DIGEST_CUDA, + sealed_manifest_hashes={" ": "c" * 64}, + ) + + +def test_allowlist_lookup_hit_returns_sealed_hashes() -> None: + """Given registered sealed hashes, When lookup hits, Then exact map is exposed.""" + registry = DigestAllowlist() + record = DigestRecord( + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + digest=DIGEST_CUDA, + sealed_manifest_hashes=SEALED_HASHES, + ) + registry.register(record) + + result = registry.lookup( + digest=DIGEST_CUDA, + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + ) + + assert isinstance(result, AllowlistHit) + assert dict(result.record.sealed_manifest_hashes) == SEALED_HASHES diff --git a/tests/unit/test_constation_bundle_seal.py b/tests/unit/test_constation_bundle_seal.py new file mode 100644 index 00000000..b7d1e893 --- /dev/null +++ b/tests/unit/test_constation_bundle_seal.py @@ -0,0 +1,286 @@ +"""TDD: pure seal_constation_bundle → prism ConstationBundle wire dict.""" + +from __future__ import annotations + +import sys +from collections.abc import Mapping +from pathlib import Path +from typing import Any, cast + +import pytest + +from base.compute.constation_types import ( + ConstationFailCode, + ConstationRunRecord, + CorroborationStatus, + FaultClass, +) +from base.compute.digest_allowlist import DigestRecord, ImageVariant +from base.master.constation.bundle_seal import seal_constation_bundle + +COMMIT = "a" * 40 +TREE = "b" * 40 +DIGEST = "sha256:" + ("1" * 64) +MANIFEST = {"src/harness.py": "c" * 64} +WIRE_MANIFEST = {"src/harness.py": "d" * 64} + +_PRISM_WIRE_KEYS = frozenset( + { + "commit_sha", + "tree_sha", + "variant", + "digest", + "work_unit_id", + "miner_hotkey", + "pod_id", + "nonce", + "signed_attestation", + "expected_sealed_manifest_hashes", + "reported_sealed_manifest_hashes", + "lium_declared_digest", + "constation_gap_budget_seconds", + "constation_observed_max_gap_seconds", + } +) + + +def _allowlist( + *, + sealed: Mapping[str, str] | None = None, +) -> DigestRecord: + return DigestRecord( + commit_sha=COMMIT, + tree_sha=TREE, + variant=ImageVariant.CUDA, + digest=DIGEST, + sealed_manifest_hashes=dict(sealed if sealed is not None else MANIFEST), + ) + + +def _run( + *, + work_unit_id: str = "wu-1", + miner_hotkey: str = "hk-miner", + pod_id: str = "pod-9", + lium: str | None = DIGEST, + gap_budget: float = 30.0, + gap_obs: float = 1.5, +) -> ConstationRunRecord: + return ConstationRunRecord( + ok=True, + reason=ConstationFailCode.OK, + fault_class=None, + miner_hotkey=miner_hotkey, + work_unit_id=work_unit_id, + pod_id=pod_id, + sidecar_digest=DIGEST, + lium_declared_digest=lium, + constation_gap_budget_seconds=gap_budget, + constation_observed_max_gap_seconds=gap_obs, + corroboration_status=CorroborationStatus.AGREE, + samples=(), + ) + + +def _sidecar_wire( + *, + sealed: Mapping[str, str] | None = WIRE_MANIFEST, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "digest": DIGEST, + "nonce": "poll-n", + "pod_id": "pod-9", + "variant": "cuda", + "build_secret_response": "ab" * 32, + } + if sealed is not None: + payload["sealed_manifest_hashes"] = dict(sealed) + return { + "payload": payload, + "signature": "ef" * 32, + "algorithm": "hmac-sha256", + "schema_version": "prism_attestation_payload.v1", + "phase": "end", + } + + +def _prism_from_dict(): + """Load constation_bundle_from_dict when prism package is on path.""" + try: + from prism_challenge.constation import constation_bundle_from_dict + + return constation_bundle_from_dict + except ImportError: + pass + root = Path(__file__).resolve().parents[2] + prism_src = root / "packages" / "challenges" / "prism" / "src" + if not prism_src.is_dir(): + return None + inserted = str(prism_src) + if inserted not in sys.path: + sys.path.insert(0, inserted) + try: + from prism_challenge.constation import constation_bundle_from_dict + + return constation_bundle_from_dict + except ImportError: + return None + + +def test_seal_happy_path_wire_fields() -> None: + # Given allowlist identity, run record, end-phase nonce, last sidecar wire + allow = _allowlist() + run = _run() + wire = _sidecar_wire() + nonce = "end-phase-nonce-abc" + + # When seal_constation_bundle assembles the prism wire dict + out = seal_constation_bundle( + allowlist_record=allow, + run_record=run, + nonce=nonce, + signed_attestation=wire, + ) + + # Then identity + binding + gap fields match sources; keys are prism wire + assert set(out) == _PRISM_WIRE_KEYS + assert out["commit_sha"] == COMMIT + assert out["tree_sha"] == TREE + assert out["variant"] == "cuda" + assert out["digest"] == DIGEST + assert out["work_unit_id"] == "wu-1" + assert out["miner_hotkey"] == "hk-miner" + assert out["pod_id"] == "pod-9" + assert out["nonce"] == nonce + assert out["signed_attestation"] == wire + assert out["expected_sealed_manifest_hashes"] == dict(MANIFEST) + assert out["reported_sealed_manifest_hashes"] == dict(WIRE_MANIFEST) + assert out["lium_declared_digest"] == DIGEST + assert out["constation_gap_budget_seconds"] == 30.0 + assert out["constation_observed_max_gap_seconds"] == 1.5 + + +def test_seal_reported_hashes_fallback_to_expected_when_absent_on_wire() -> None: + # Given sidecar wire without sealed_manifest_hashes + allow = _allowlist() + run = _run() + wire = _sidecar_wire(sealed=None) + assert "sealed_manifest_hashes" not in wire["payload"] + + # When sealed + out = seal_constation_bundle( + allowlist_record=allow, + run_record=run, + nonce="n1", + signed_attestation=wire, + ) + + # Then reported copies expected (allowlist sealed surface) + assert out["reported_sealed_manifest_hashes"] == dict(MANIFEST) + assert out["expected_sealed_manifest_hashes"] == dict(MANIFEST) + + +def test_seal_missing_signed_attestation_raises_bundle_incomplete() -> None: + # Given no last good sidecar wire + # When seal runs + # Then fail-closed with BUNDLE_INCOMPLETE + with pytest.raises(ValueError, match="BUNDLE_INCOMPLETE"): + seal_constation_bundle( + allowlist_record=_allowlist(), + run_record=_run(), + nonce="n1", + signed_attestation=None, + ) + + +def test_seal_empty_nonce_raises_bundle_incomplete() -> None: + with pytest.raises(ValueError, match="BUNDLE_INCOMPLETE"): + seal_constation_bundle( + allowlist_record=_allowlist(), + run_record=_run(), + nonce=" ", + signed_attestation=_sidecar_wire(), + ) + + +def test_seal_blank_binding_fields_raise() -> None: + with pytest.raises(ValueError, match="BUNDLE_INCOMPLETE"): + seal_constation_bundle( + allowlist_record=_allowlist(), + run_record=_run(work_unit_id=""), + nonce="n1", + signed_attestation=_sidecar_wire(), + ) + + +def test_seal_lium_declared_digest_none_allowed() -> None: + out = seal_constation_bundle( + allowlist_record=_allowlist(), + run_record=_run(lium=None), + nonce="n1", + signed_attestation=_sidecar_wire(), + ) + assert out["lium_declared_digest"] is None + + +def test_seal_does_not_mutate_inputs() -> None: + allow = _allowlist() + run = _run() + wire = _sidecar_wire() + wire_before = { + "payload": dict(wire["payload"]), + "signature": wire["signature"], + "algorithm": wire["algorithm"], + "schema_version": wire["schema_version"], + "phase": wire["phase"], + } + out = seal_constation_bundle( + allowlist_record=allow, + run_record=run, + nonce="n1", + signed_attestation=wire, + ) + out["nonce"] = "mutated" + nested = cast(dict[str, object], out["expected_sealed_manifest_hashes"]) + nested["x"] = "y" + assert wire == wire_before + assert "x" not in allow.sealed_manifest_hashes + + +def test_seal_roundtrip_prism_constation_bundle_from_dict() -> None: + # Given a sealed wire dict + from_dict = _prism_from_dict() + if from_dict is None: + pytest.skip("prism_challenge not importable") + + out = seal_constation_bundle( + allowlist_record=_allowlist(), + run_record=_run(), + nonce="roundtrip-nonce", + signed_attestation=_sidecar_wire(), + ) + + # When parsed by prism boundary + bundle = from_dict(out) + + # Then all identity/binding/gap fields survive + assert bundle.commit_sha == COMMIT + assert bundle.tree_sha == TREE + assert bundle.variant == "cuda" + assert bundle.digest == DIGEST + assert bundle.work_unit_id == "wu-1" + assert bundle.miner_hotkey == "hk-miner" + assert bundle.pod_id == "pod-9" + assert bundle.nonce == "roundtrip-nonce" + assert bundle.signed_attestation == out["signed_attestation"] + assert dict(bundle.expected_sealed_manifest_hashes) == dict(MANIFEST) + assert dict(bundle.reported_sealed_manifest_hashes) == dict(WIRE_MANIFEST) + assert bundle.lium_declared_digest == DIGEST + assert bundle.constation_gap_budget_seconds == 30.0 + assert bundle.constation_observed_max_gap_seconds == 1.5 + + +def test_bundle_incomplete_code_exists_for_callers() -> None: + # Adjacent: fail code available for infra attribution if callers catch + assert ConstationFailCode.BUNDLE_INCOMPLETE.value == "bundle_incomplete" + assert FaultClass.INFRA.value == "infra_fault" diff --git a/tests/unit/test_constation_main_wiring.py b/tests/unit/test_constation_main_wiring.py new file mode 100644 index 00000000..434eceb4 --- /dev/null +++ b/tests/unit/test_constation_main_wiring.py @@ -0,0 +1,259 @@ +"""T9: master lifespan wires custody + pod_binding + orchestrator + hook. + +Regression guards: +- build_constation_router receives pod_binding= (register_miner_key not 503) +- custody master key load fail-closed +- WorkerReconciliationService invokes constation_hook before forward_result +""" + +from __future__ import annotations + +import ast +import inspect +from collections.abc import Mapping +from pathlib import Path +from typing import Any, cast +from unittest.mock import AsyncMock + +import pytest +from cryptography.fernet import Fernet + +from base.compute.constation_custody import generate_custody_master_key +from base.config.settings import ConstationSettings, Settings +from base.db.models import WorkAssignment, WorkerAssignment +from base.master.constation.custody_keys import ( + build_constation_runtime, + load_custody_master_key, + poller_config_from_settings, +) +from base.master.worker_reconciliation import WorkerReconciliationService + + +def test_main_passes_pod_binding_into_build_constation_router() -> None: + """Given main.py call site, When AST-inspected, Then pod_binding kwarg present.""" + import base.cli_app.main as main_mod + + source = inspect.getsource(main_mod) + tree = ast.parse(source) + hits: list[ast.Call] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + name = ( + func.id + if isinstance(func, ast.Name) + else func.attr + if isinstance(func, ast.Attribute) + else None + ) + if name != "build_constation_router": + continue + hits.append(node) + assert hits, "build_constation_router call missing from main" + for call in hits: + kw_names = {kw.arg for kw in call.keywords if kw.arg is not None} + assert "pod_binding" in kw_names, ( + "build_constation_router must receive pod_binding= " + f"(got keywords {sorted(kw_names)})" + ) + + +def test_load_custody_master_key_missing_returns_none() -> None: + """Given no custody key, When load, Then None (fail-closed).""" + assert load_custody_master_key(Settings()) is None + assert ( + load_custody_master_key(Settings(constation=ConstationSettings(enabled=True))) + is None + ) + assert ( + load_custody_master_key( + Settings(constation=ConstationSettings(custody_master_key="")) + ) + is None + ) + assert ( + load_custody_master_key( + Settings(constation=ConstationSettings(custody_master_key=" ")) + ) + is None + ) + + +def test_load_custody_master_key_inline_and_file(tmp_path: Path) -> None: + """Given inline or file Fernet key, When load, Then raw key bytes.""" + key = generate_custody_master_key() + text = key.decode("ascii") + settings_inline = Settings(constation=ConstationSettings(custody_master_key=text)) + assert load_custody_master_key(settings_inline) == key + + key_path = tmp_path / "custody.key" + key_path.write_text(text, encoding="utf-8") + settings_file = Settings( + constation=ConstationSettings(custody_master_key_file=key_path) + ) + assert load_custody_master_key(settings_file) == key + + +def test_build_constation_runtime_disabled_skips_services() -> None: + """Given enabled=False, When build runtime, Then binding/orch None.""" + key = generate_custody_master_key().decode("ascii") + runtime = build_constation_runtime( + Settings(constation=ConstationSettings(enabled=False, custody_master_key=key)), + nonce_service=object(), + bundle_store=object(), + ) + assert runtime.pod_binding is None + assert runtime.orchestrator is None + assert runtime.enabled is False + + +def test_build_constation_runtime_enabled_missing_key_fail_closed() -> None: + """Given enabled=True without key, When build, Then None services (no crash).""" + runtime = build_constation_runtime( + Settings(constation=ConstationSettings(enabled=True)), + nonce_service=object(), + bundle_store=object(), + ) + assert runtime.enabled is True + assert runtime.pod_binding is None + assert runtime.orchestrator is None + + +def test_build_constation_runtime_enabled_with_key() -> None: + """Given enabled + valid key, When build, Then binding + orchestrator ready.""" + key = generate_custody_master_key().decode("ascii") + nonce = object() + store = object() + runtime = build_constation_runtime( + Settings( + constation=ConstationSettings( + enabled=True, + custody_master_key=key, + gap_budget_seconds=12.0, + sidecar_internal_port=9999, + ) + ), + nonce_service=nonce, + bundle_store=store, + ) + assert runtime.pod_binding is not None + assert runtime.orchestrator is not None + assert runtime.orchestrator.nonce_service is nonce + assert runtime.orchestrator.bundle_store is store + assert runtime.orchestrator.sidecar_internal_port == 9999 + assert runtime.orchestrator.poller_config.gap_budget_seconds == 12.0 + # Fernet accepts the loaded master key + Fernet(runtime.pod_binding.custody.master_key) + + +def test_poller_config_from_settings_maps_fields() -> None: + """Given ConstationSettings, When map, Then PollerConfig fields match.""" + cs = ConstationSettings( + gap_budget_seconds=11.0, + min_interval_seconds=2.0, + max_interval_seconds=9.0, + max_polls=40, + ) + cfg = poller_config_from_settings(cs) + assert cfg.gap_budget_seconds == 11.0 + assert cfg.min_interval_seconds == 2.0 + assert cfg.max_interval_seconds == 9.0 + assert cfg.max_polls == 40 + assert cfg.max_cost_units == 40.0 + + +class _FakePrimary: + challenge_slug = "prism" + work_unit_id = "wu-hook-1" + submission_ref = "sub-1" + payload: dict[str, Any] = {"required_digest": "sha256:" + ("a" * 64)} + + +class _FakeWinner: + miner_hotkey = "hk-hook" + result_payload: dict[str, Any] = {"ok": True} + + +class _Forwarder: + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + + async def forward_result( + self, + *, + challenge_slug: str, + work_unit_id: str, + submission_ref: str, + result_payload: Mapping[str, Any], + ) -> None: + self.calls.append( + { + "challenge_slug": challenge_slug, + "work_unit_id": work_unit_id, + "submission_ref": submission_ref, + "result_payload": dict(result_payload), + } + ) + + +@pytest.mark.asyncio +async def test_reconciliation_constation_hook_called_before_forward() -> None: + """Given hook, When _forward, Then hook runs before forward_result.""" + order: list[str] = [] + seen: dict[str, Any] = {} + + async def _hook( + *, + work_unit_id: str, + miner_hotkey: str, + metadata: Mapping[str, Any], + ) -> None: + order.append("hook") + seen["work_unit_id"] = work_unit_id + seen["miner_hotkey"] = miner_hotkey + seen["metadata"] = dict(metadata) + + class _OrderedForwarder: + async def forward_result( + self, + *, + challenge_slug: str, + work_unit_id: str, + submission_ref: str, + result_payload: Mapping[str, Any], + ) -> None: + order.append("forward") + seen["forward_unit"] = work_unit_id + + svc = WorkerReconciliationService( + session_factory=AsyncMock(), # unused by _forward + result_forwarder=_OrderedForwarder(), + constation_hook=_hook, + ) + ok = await svc._forward( # noqa: SLF001 + cast(WorkAssignment, _FakePrimary()), + cast(WorkerAssignment, _FakeWinner()), + ) + assert ok is True + assert order == ["hook", "forward"] + assert seen["work_unit_id"] == "wu-hook-1" + assert seen["miner_hotkey"] == "hk-hook" + assert seen["metadata"]["required_digest"].startswith("sha256:") + assert seen["forward_unit"] == "wu-hook-1" + + +@pytest.mark.asyncio +async def test_reconciliation_without_hook_still_forwards() -> None: + """Given no hook, When _forward, Then forward still succeeds.""" + forwarder = _Forwarder() + svc = WorkerReconciliationService( + session_factory=AsyncMock(), + result_forwarder=forwarder, + ) + ok = await svc._forward( # noqa: SLF001 + cast(WorkAssignment, _FakePrimary()), + cast(WorkerAssignment, _FakeWinner()), + ) + assert ok is True + assert len(forwarder.calls) == 1 diff --git a/tests/unit/test_constation_orchestrator.py b/tests/unit/test_constation_orchestrator.py new file mode 100644 index 00000000..dd3dda77 --- /dev/null +++ b/tests/unit/test_constation_orchestrator.py @@ -0,0 +1,404 @@ +"""TDD: ProductionConstationOrchestrator — issue nonce → run → seal → put (B2). + +B2: orchestrator MUST issue nonces and MUST NEVER consume. After seal, the +end-phase nonce remains first-consumable for prism ingest. +""" + +from __future__ import annotations + +import inspect +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from typing import Any + +import pytest + +from base.compute.attestation_nonce import ( + AttestationNonceService, + NonceBinding, + NonceConsumeHit, + NonceConsumeMiss, + NonceConsumeReason, +) +from base.compute.constation_custody import LiumKeyCustody, generate_custody_master_key +from base.compute.constation_poller import PollerConfig +from base.compute.constation_runner import ConstationRunRequest +from base.compute.constation_types import ( + ConstationFailCode, + ConstationRunRecord, + CorroborationStatus, + FaultClass, +) +from base.compute.digest_allowlist import ImageVariant +from base.master.constation.bundle_store import ConstationBundleStore +from base.master.constation.orchestrator import ( + ConstationOrchestrationRequest, + ConstationOrchestrationResult, + ProductionConstationOrchestrator, +) +from base.master.constation.pod_binding import MinerPodBinding + +COMMIT = "a" * 40 +TREE = "b" * 40 +DIGEST = "sha256:" + ("c" * 64) +MANIFEST = {"src/harness.py": "d" * 64} +HOTKEY = "5MinerOrchTestHotkey000000000000000001" +POD = "pod-orch-001" +WORK_UNIT = "wu-orch-001" +T0 = datetime(2026, 7, 27, 5, 0, 0, tzinfo=UTC) +WIRE = { + "payload": { + "digest": DIGEST, + "nonce": "will-be-overwritten-in-assert", + "pod_id": POD, + "variant": "cuda", + "build_secret_response": "ab" * 32, + "sealed_manifest_hashes": dict(MANIFEST), + }, + "signature": "ef" * 32, + "algorithm": "hmac-sha256", + "schema_version": "prism_attestation_payload.v1", + "phase": "end", +} + + +def _ok_record( + *, + work_unit_id: str = WORK_UNIT, + miner_hotkey: str = HOTKEY, + pod_id: str = POD, +) -> ConstationRunRecord: + return ConstationRunRecord( + ok=True, + reason=ConstationFailCode.OK, + fault_class=None, + miner_hotkey=miner_hotkey, + work_unit_id=work_unit_id, + pod_id=pod_id, + sidecar_digest=DIGEST, + lium_declared_digest=DIGEST, + constation_gap_budget_seconds=30.0, + constation_observed_max_gap_seconds=1.0, + corroboration_status=CorroborationStatus.AGREE, + samples=(), + ) + + +def _fail_record( + *, + reason: ConstationFailCode = ConstationFailCode.SIDECAR_ATTEST_FAILED, +) -> ConstationRunRecord: + return ConstationRunRecord( + ok=False, + reason=reason, + fault_class=FaultClass.INFRA, + miner_hotkey=HOTKEY, + work_unit_id=WORK_UNIT, + pod_id=POD, + sidecar_digest=None, + lium_declared_digest=None, + constation_gap_budget_seconds=30.0, + constation_observed_max_gap_seconds=0.0, + corroboration_status=CorroborationStatus.NOT_EVALUATED, + samples=(), + ) + + +@dataclass +class _FakeRunner: + """Stand-in ConstationRunner: issues via poll_nonce_fn, sets last_signed_wire.""" + + poll_nonce_fn: Callable[[], str] | None = None + last_signed_wire: Mapping[str, Any] | None = field(default=None, init=False) + outcome: ConstationRunRecord = field(default_factory=_ok_record) + wire: Mapping[str, Any] | None = field(default_factory=lambda: dict(WIRE)) + run_calls: int = 0 + last_request: ConstationRunRequest | None = None + phases_to_issue: tuple[str, ...] = ("start", "mid", "end") + + async def run(self, request: ConstationRunRequest) -> ConstationRunRecord: + self.run_calls += 1 + self.last_request = request + if self.outcome.ok and self.poll_nonce_fn is not None: + for _phase in self.phases_to_issue: + self.poll_nonce_fn() + self.last_signed_wire = dict(self.wire) if self.wire is not None else None + else: + self.last_signed_wire = None + # Align binding fields with the request the orchestrator built. + if self.outcome.ok: + return _ok_record( + work_unit_id=request.work_unit_id, + miner_hotkey=request.miner_hotkey, + pod_id=request.pod_id, + ) + return self.outcome + + +def _binding_store(*, hotkey: str = HOTKEY, pod_id: str = POD) -> MinerPodBinding: + custody = LiumKeyCustody(master_key=generate_custody_master_key()) + custody.store_probed_key(miner_hotkey=hotkey, api_key="lium-test-key-not-logged") + binding = MinerPodBinding(custody=custody) + binding._instance_by_hotkey[hotkey.strip()] = pod_id # noqa: SLF001 — test seam + return binding + + +def _request(**overrides: Any) -> ConstationOrchestrationRequest: + base: dict[str, Any] = { + "work_unit_id": WORK_UNIT, + "miner_hotkey": HOTKEY, + "required_digest": DIGEST, + "commit_sha": COMMIT, + "tree_sha": TREE, + "variant": ImageVariant.CUDA, + "sealed_manifest_hashes": dict(MANIFEST), + "duration_seconds": 5.0, + } + base.update(overrides) + return ConstationOrchestrationRequest(**base) + + +def _orchestrator( + *, + pod_binding: MinerPodBinding | None = None, + nonce_service: AttestationNonceService | None = None, + bundle_store: ConstationBundleStore | None = None, + fake: _FakeRunner | None = None, +) -> tuple[ + ProductionConstationOrchestrator, + AttestationNonceService, + ConstationBundleStore, + _FakeRunner, +]: + binding = pod_binding if pod_binding is not None else _binding_store() + nonces = ( + nonce_service + if nonce_service is not None + else AttestationNonceService(ttl=timedelta(hours=1), now_fn=lambda: T0) + ) + store = bundle_store if bundle_store is not None else ConstationBundleStore() + runner = fake if fake is not None else _FakeRunner() + + def runner_factory(**kwargs: Any) -> _FakeRunner: + runner.poll_nonce_fn = kwargs.get("poll_nonce_fn") + # Preserve other kwargs for API compatibility with real ConstationRunner + return runner + + orch = ProductionConstationOrchestrator( + pod_binding=binding, + nonce_service=nonces, + bundle_store=store, + poller_config=PollerConfig(), + now_fn=lambda: 0.0, + sleep_fn=_async_noop_sleep, + rng_fn=lambda: 0.0, + runner_factory=runner_factory, + ) + return orch, nonces, store, runner + + +async def _async_noop_sleep(_seconds: float) -> None: + return None + + +@pytest.mark.asyncio +async def test_happy_path_seals_and_puts_bundle() -> None: + """S1: binding ok → issue nonces → runner ok → seal → bundle_store.put.""" + orch, nonces, store, runner = _orchestrator() + + result = await orch.run(_request()) + + assert isinstance(result, ConstationOrchestrationResult) + assert result.ok is True + assert result.reason is ConstationFailCode.OK + assert runner.run_calls == 1 + assert runner.last_request is not None + assert runner.last_request.required_digest == DIGEST + assert runner.last_request.pod_id == POD + assert runner.last_request.work_unit_id == WORK_UNIT + + stored = store.get(WORK_UNIT) + assert stored is not None + assert stored["work_unit_id"] == WORK_UNIT + assert stored["miner_hotkey"] == HOTKEY + assert stored["pod_id"] == POD + assert stored["digest"] == DIGEST + assert stored["commit_sha"] == COMMIT + assert stored["tree_sha"] == TREE + assert stored["variant"] == "cuda" + assert stored["nonce"] == result.end_phase_nonce + assert result.end_phase_nonce is not None + assert stored["signed_attestation"] == dict(WIRE) + # Three poll phases each issued a nonce; end-phase is the last. + assert len(nonces.snapshot().records) == 3 + assert result.end_phase_nonce == nonces.snapshot().records[-1].nonce + + +@pytest.mark.asyncio +async def test_runner_fail_does_not_put_bundle() -> None: + """S2: runner.ok False → do NOT put bundle (prism missing_constation_bundle).""" + fake = _FakeRunner(outcome=_fail_record()) + orch, _nonces, store, runner = _orchestrator(fake=fake) + + result = await orch.run(_request()) + + assert result.ok is False + assert result.reason is ConstationFailCode.SIDECAR_ATTEST_FAILED + assert store.get(WORK_UNIT) is None + assert result.bundle is None + assert runner.run_calls == 1 + + +@pytest.mark.asyncio +async def test_b2_end_phase_nonce_first_consumable_after_seal() -> None: + """B2: AFTER seal, consume once → Hit; second → already_consumed. + + If orchestrator had consumed, first post-seal consume would miss. + """ + orch, nonces, store, _runner = _orchestrator() + + result = await orch.run(_request()) + + assert result.ok is True + assert result.end_phase_nonce is not None + binding = NonceBinding(work_unit_id=WORK_UNIT, miner_hotkey=HOTKEY, pod_id=POD) + stored = store.get(WORK_UNIT) + assert stored is not None + assert stored["nonce"] == result.end_phase_nonce + + first = nonces.consume(result.end_phase_nonce, binding) + assert isinstance(first, NonceConsumeHit), ( + f"B2 violated: end-phase nonce not first-consumable after seal " + f"(got {first!r}); orchestrator must not consume" + ) + + second = nonces.consume(result.end_phase_nonce, binding) + assert isinstance(second, NonceConsumeMiss) + assert second.reason is NonceConsumeReason.ALREADY_CONSUMED + + +@pytest.mark.asyncio +async def test_b2_orchestrator_source_never_calls_consume() -> None: + """Static guard: orchestrator module must not invoke nonce consume.""" + from base.master.constation import orchestrator as orch_mod + + src = inspect.getsource(orch_mod) + # Allow the word only in comments/docstrings about NEVER consume — ban call form. + assert ".consume(" not in src + assert "nonce_service.consume" not in src + + +@pytest.mark.asyncio +async def test_missing_binding_fail_closed_no_put() -> None: + """S4: custody/binding missing → fail closed, no runner, no put.""" + custody = LiumKeyCustody(master_key=generate_custody_master_key()) + empty = MinerPodBinding(custody=custody) + store = ConstationBundleStore() + fake = _FakeRunner() + orch, _n, store, runner = _orchestrator( + pod_binding=empty, bundle_store=store, fake=fake + ) + + result = await orch.run(_request()) + + assert result.ok is False + assert result.reason is ConstationFailCode.KEY_NOT_REGISTERED + assert store.get(WORK_UNIT) is None + assert runner.run_calls == 0 + + +@pytest.mark.asyncio +async def test_request_pod_id_override_and_instance_id_alias() -> None: + """DTO accepts pod_id or instance_id; override beats binding lookup.""" + orch, _n, store, runner = _orchestrator() + other_pod = "pod-override-99" + + result = await orch.run(_request(pod_id=other_pod)) + + assert result.ok is True + assert runner.last_request is not None + assert runner.last_request.pod_id == other_pod + stored = store.get(WORK_UNIT) + assert stored is not None + assert stored["pod_id"] == other_pod + + # instance_id alias when pod_id omitted + fake2 = _FakeRunner() + orch2, _n2, store2, runner2 = _orchestrator(fake=fake2) + result2 = await orch2.run(_request(pod_id=None, instance_id="pod-via-instance")) + assert result2.ok is True + assert runner2.last_request is not None + assert runner2.last_request.pod_id == "pod-via-instance" + + +@pytest.mark.asyncio +async def test_missing_signed_wire_fail_closed_no_put() -> None: + """Fail-closed: runner ok but no last_signed_wire → no bundle put.""" + fake = _FakeRunner(wire=None) + # Force ok path without wire: custom run + original_run = fake.run + + async def run_ok_no_wire(request: ConstationRunRequest) -> ConstationRunRecord: + fake.run_calls += 1 + fake.last_request = request + if fake.poll_nonce_fn is not None: + fake.poll_nonce_fn() + fake.last_signed_wire = None + return _ok_record() + + fake.run = run_ok_no_wire # type: ignore[method-assign] + del original_run + + orch, _n, store, _r = _orchestrator(fake=fake) + result = await orch.run(_request()) + + assert result.ok is False + assert store.get(WORK_UNIT) is None + + +def test_orchestration_request_is_explicit_dto_no_hidden_globals() -> None: + """ConstationOrchestrationRequest exposes all required fields explicitly.""" + req = _request(instance_id="pod-x", duration_seconds=12.5) + assert req.work_unit_id == WORK_UNIT + assert req.miner_hotkey == HOTKEY + assert req.required_digest == DIGEST + assert req.commit_sha == COMMIT + assert req.tree_sha == TREE + assert req.variant == ImageVariant.CUDA + assert dict(req.sealed_manifest_hashes) == MANIFEST + assert req.duration_seconds == 12.5 + assert req.instance_id == "pod-x" + + +@dataclass +class _AsyncNonceService: + """Durable-shaped async issuer for B2 / async issue path.""" + + inner: AttestationNonceService + + async def issue(self, binding: NonceBinding) -> Any: + return self.inner.issue(binding) + + async def consume(self, nonce: str, binding: NonceBinding) -> Any: + return self.inner.consume(nonce, binding) + + +@pytest.mark.asyncio +async def test_b2_async_nonce_service_end_phase_still_first_consumable() -> None: + """Async durable-shaped issuer: still issue-only; consume Hit after seal.""" + inner = AttestationNonceService(ttl=timedelta(hours=1), now_fn=lambda: T0) + async_svc = _AsyncNonceService(inner=inner) + orch, _n, store, _r = _orchestrator(nonce_service=async_svc) # type: ignore[arg-type] + + result = await orch.run(_request()) + + assert result.ok is True + assert result.end_phase_nonce is not None + assert store.get(WORK_UNIT) is not None + binding = NonceBinding(work_unit_id=WORK_UNIT, miner_hotkey=HOTKEY, pod_id=POD) + first = await async_svc.consume(result.end_phase_nonce, binding) + assert isinstance(first, NonceConsumeHit) + second = await async_svc.consume(result.end_phase_nonce, binding) + assert isinstance(second, NonceConsumeMiss) + assert second.reason is NonceConsumeReason.ALREADY_CONSUMED diff --git a/tests/unit/test_constation_pod.py b/tests/unit/test_constation_pod.py new file mode 100644 index 00000000..2f8aa29f --- /dev/null +++ b/tests/unit/test_constation_pod.py @@ -0,0 +1,160 @@ +"""TDD tests for pure Lium pod hotkey + running-status helpers. + +Fail-closed: never invent miner_hotkey; only top-level status == RUNNING. +""" + +from __future__ import annotations + +from typing import Any + +from base.compute.constation_pod import ( + assert_pod_bound, + extract_miner_hotkey, + pod_is_running, +) +from base.compute.constation_types import ConstationFailCode, ConstationVerdict + +_HOTKEY = "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty" +_OTHER = "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY" + + +def _pod( + *, + miner_hotkey: object = _HOTKEY, + status: object = "RUNNING", + executor: object | None = ..., # type: ignore[assignment] + extra: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Minimal Lium PodDetailResponse-shaped mapping (mirrors sidecar tests).""" + if executor is ...: + executor_val: object = { + "executor_ip_address": "1.2.3.4", + "miner_hotkey": miner_hotkey, + "validator_hotkey": "5DAAnrj7VHTznn2AWBemMuyBwZWs6FNFjdyVXUeYum3PTXFy", + } + else: + executor_val = executor + raw: dict[str, Any] = { + "id": "pod-1", + "status": status, + "executor": executor_val, + } + if extra: + raw.update(extra) + return raw + + +def test_extract_miner_hotkey_when_nested_executor_present() -> None: + # Given executor.miner_hotkey is a non-blank string + # When extract_miner_hotkey runs + result = extract_miner_hotkey(_pod(miner_hotkey=f" {_HOTKEY} ")) + # Then stripped hotkey is returned + assert result == _HOTKEY + + +def test_extract_miner_hotkey_when_executor_missing() -> None: + # Given no executor mapping + # When extract_miner_hotkey runs + assert extract_miner_hotkey(_pod(executor=None)) is None + assert extract_miner_hotkey({"id": "pod-1", "status": "RUNNING"}) is None + assert extract_miner_hotkey(_pod(executor="not-a-map")) is None + + +def test_extract_miner_hotkey_when_hotkey_absent_or_blank() -> None: + # Given executor without usable miner_hotkey + # When extract_miner_hotkey runs + # Then fail-closed None (never invent) + assert extract_miner_hotkey(_pod(miner_hotkey=None)) is None + assert extract_miner_hotkey(_pod(miner_hotkey="")) is None + assert extract_miner_hotkey(_pod(miner_hotkey=" ")) is None + assert extract_miner_hotkey(_pod(miner_hotkey=12345)) is None + # Top-level miner_hotkey must not be used as a fallback + assert ( + extract_miner_hotkey( + _pod( + executor={"executor_ip_address": "1.2.3.4"}, + extra={"miner_hotkey": _HOTKEY}, + ) + ) + is None + ) + + +def test_pod_is_running_when_status_running() -> None: + # Given top-level status RUNNING (any case / padding) + assert pod_is_running(_pod(status="RUNNING")) is True + assert pod_is_running(_pod(status="running")) is True + assert pod_is_running(_pod(status=" Running ")) is True + + +def test_pod_is_running_when_stopped_or_absent() -> None: + # Given non-RUNNING statuses or missing field + assert pod_is_running(_pod(status="STOPPED")) is False + assert pod_is_running(_pod(status="PENDING")) is False + assert pod_is_running(_pod(status="FAILED")) is False + assert pod_is_running(_pod(status="CREATION_FAILED")) is False + assert pod_is_running(_pod(status="BROKEN")) is False + assert pod_is_running(_pod(status="")) is False + assert pod_is_running(_pod(status=None)) is False + assert ( + pod_is_running({"id": "pod-1", "executor": {"miner_hotkey": _HOTKEY}}) is False + ) + assert pod_is_running(_pod(status=1)) is False + + +def test_assert_pod_bound_ok_when_hotkey_matches_and_running() -> None: + # Given matching hotkey and RUNNING + # When assert_pod_bound runs + result = assert_pod_bound(pod_raw=_pod(), expected_hotkey=_HOTKEY) + # Then typed ok verdict + assert isinstance(result, ConstationVerdict) + assert result.ok is True + assert result.reason is ConstationFailCode.OK + assert bool(result) is True + + +def test_assert_pod_bound_fails_hotkey_mismatch() -> None: + # Given running pod bound to a different miner + result = assert_pod_bound( + pod_raw=_pod(miner_hotkey=_OTHER), expected_hotkey=_HOTKEY + ) + assert result.ok is False + assert result.reason is ConstationFailCode.POD_HOTKEY_MISMATCH + assert bool(result) is False + + +def test_assert_pod_bound_fails_when_hotkey_absent() -> None: + # Given missing nested hotkey — treat as mismatch (fail-closed) + result = assert_pod_bound( + pod_raw=_pod(executor={"executor_ip_address": "1.2.3.4"}), + expected_hotkey=_HOTKEY, + ) + assert result.ok is False + assert result.reason is ConstationFailCode.POD_HOTKEY_MISMATCH + + +def test_assert_pod_bound_fails_when_not_running() -> None: + # Given matching hotkey but STOPPED + result = assert_pod_bound( + pod_raw=_pod(status="STOPPED"), + expected_hotkey=_HOTKEY, + ) + assert result.ok is False + assert result.reason is ConstationFailCode.POD_NOT_RUNNING + + +def test_assert_pod_bound_mismatch_precedes_not_running() -> None: + # Given wrong hotkey AND not running — hotkey check wins first + result = assert_pod_bound( + pod_raw=_pod(miner_hotkey=_OTHER, status="STOPPED"), + expected_hotkey=_HOTKEY, + ) + assert result.ok is False + assert result.reason is ConstationFailCode.POD_HOTKEY_MISMATCH + + +def test_assert_pod_bound_strips_expected_hotkey() -> None: + # Given expected_hotkey with surrounding whitespace + result = assert_pod_bound(pod_raw=_pod(), expected_hotkey=f" {_HOTKEY} ") + assert result.ok is True + assert result.reason is ConstationFailCode.OK diff --git a/tests/unit/test_constation_pod_binding.py b/tests/unit/test_constation_pod_binding.py new file mode 100644 index 00000000..8429fbba --- /dev/null +++ b/tests/unit/test_constation_pod_binding.py @@ -0,0 +1,399 @@ +"""TDD tests for miner Lium API key + instance_id pod binding (T6a domain). + +Fail-closed: probe → get_pod_raw → assert_pod_bound → store encrypted key + id. +Never logs api_key. No HTTP routes. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from typing import Any + +import pytest + +from base.compute.constation_custody import ( + LiumKeyCustody, + generate_custody_master_key, +) +from base.compute.constation_types import ConstationFailCode, FaultClass +from base.compute.lium import ( + LiumAuthError, + LiumError, + LiumNotFoundError, + LiumPodRead, + LiumRateLimitError, +) +from base.master.constation.pod_binding import MinerPodBinding + +HOTKEY = "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty" +OTHER = "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY" +INSTANCE = "pod-bind-001" +API_KEY = "lium-bind-test-key-NEVER-LOG-THIS-VALUE-abc" + + +def _pod_raw( + *, + miner_hotkey: object = HOTKEY, + status: object = "RUNNING", + pod_id: str = INSTANCE, +) -> dict[str, Any]: + return { + "id": pod_id, + "status": status, + "executor": { + "executor_ip_address": "1.2.3.4", + "miner_hotkey": miner_hotkey, + }, + } + + +@dataclass +class ScriptedBindClient: + """LiumClient stand-in: balance probe + get_pod_raw.""" + + pod_raw: dict[str, Any] = field(default_factory=_pod_raw) + probe_error: BaseException | None = None + pod_error: BaseException | None = None + balance_calls: int = 0 + pod_calls: int = 0 + last_pod_id: str | None = None + api_key_seen: str | None = None + + async def balance(self) -> float: + self.balance_calls += 1 + if self.probe_error is not None: + raise self.probe_error + return 1.0 + + async def get_pod_raw(self, pod_id: str) -> LiumPodRead: + self.pod_calls += 1 + self.last_pod_id = pod_id + if self.pod_error is not None: + raise self.pod_error + raw = self.pod_raw + return LiumPodRead( + pod_id=str(raw.get("id") or pod_id), + template_id="tmpl-1", + docker_image_digest=None, + raw=raw, + ) + + +def _binding( + client: ScriptedBindClient, +) -> MinerPodBinding: + def factory(api_key: str) -> ScriptedBindClient: + client.api_key_seen = api_key + return client + + custody = LiumKeyCustody( + master_key=generate_custody_master_key(), + client_factory=factory, # type: ignore[arg-type] + ) + return MinerPodBinding(custody=custody) + + +# --------------------------------------------------------------------------- +# S1 happy path +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_register_binds_key_and_instance_when_pod_running( + caplog: pytest.LogCaptureFixture, +) -> None: + # Given a valid key and RUNNING pod owned by miner hotkey + client = ScriptedBindClient(pod_raw=_pod_raw()) + binding = _binding(client) + + # When register runs + with caplog.at_level(logging.DEBUG): + verdict = await binding.register( + miner_hotkey=HOTKEY, + api_key=API_KEY, + instance_id=INSTANCE, + ) + + # Then ok + encrypted key + instance_id stored; api_key never logged + assert verdict.ok is True + assert verdict.reason is ConstationFailCode.OK + assert binding.has_binding(HOTKEY) is True + assert binding.get_instance_id(HOTKEY) == INSTANCE + assert binding.custody.has_key(HOTKEY) is True + assert binding.custody.unlock_api_key(HOTKEY) == API_KEY + assert client.balance_calls >= 1 + assert client.pod_calls == 1 + assert client.last_pod_id == INSTANCE + assert API_KEY not in caplog.text + assert API_KEY not in repr(binding) + assert API_KEY not in str(binding) + blob = binding.custody.export_encrypted()[HOTKEY] + assert API_KEY.encode() not in blob + + +@pytest.mark.asyncio +async def test_register_strips_whitespace_on_inputs() -> None: + client = ScriptedBindClient(pod_raw=_pod_raw()) + binding = _binding(client) + + verdict = await binding.register( + miner_hotkey=f" {HOTKEY} ", + api_key=f" {API_KEY} ", + instance_id=f" {INSTANCE} ", + ) + + assert verdict.ok is True + assert binding.get_instance_id(HOTKEY) == INSTANCE + assert binding.custody.unlock_api_key(HOTKEY) == API_KEY + assert client.last_pod_id == INSTANCE + + +# --------------------------------------------------------------------------- +# S2 probe failures — fail closed, store nothing +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_register_probe_401_fail_closed_stores_nothing( + caplog: pytest.LogCaptureFixture, +) -> None: + client = ScriptedBindClient( + probe_error=LiumAuthError("401", status_code=401), + ) + binding = _binding(client) + + with caplog.at_level(logging.DEBUG): + verdict = await binding.register( + miner_hotkey=HOTKEY, + api_key=API_KEY, + instance_id=INSTANCE, + ) + + assert verdict.ok is False + assert verdict.reason is ConstationFailCode.LIUM_AUTH_REVOKED + assert verdict.fault_class is FaultClass.MINER + assert binding.has_binding(HOTKEY) is False + assert binding.custody.has_key(HOTKEY) is False + assert client.pod_calls == 0 + assert API_KEY not in caplog.text + + +@pytest.mark.asyncio +async def test_register_probe_other_error_is_probe_failed() -> None: + client = ScriptedBindClient(probe_error=LiumError("timeout")) + binding = _binding(client) + + verdict = await binding.register( + miner_hotkey=HOTKEY, + api_key=API_KEY, + instance_id=INSTANCE, + ) + + assert verdict.ok is False + assert verdict.reason is ConstationFailCode.PROBE_FAILED + assert binding.has_binding(HOTKEY) is False + assert binding.custody.has_key(HOTKEY) is False + assert client.pod_calls == 0 + + +# --------------------------------------------------------------------------- +# S3 / S4 pod bind failures — key must not be stored +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_register_pod_hotkey_mismatch_stores_nothing() -> None: + client = ScriptedBindClient(pod_raw=_pod_raw(miner_hotkey=OTHER)) + binding = _binding(client) + + verdict = await binding.register( + miner_hotkey=HOTKEY, + api_key=API_KEY, + instance_id=INSTANCE, + ) + + assert verdict.ok is False + assert verdict.reason is ConstationFailCode.POD_HOTKEY_MISMATCH + assert binding.has_binding(HOTKEY) is False + assert binding.custody.has_key(HOTKEY) is False + + +@pytest.mark.asyncio +async def test_register_pod_not_running_stores_nothing() -> None: + client = ScriptedBindClient(pod_raw=_pod_raw(status="STOPPED")) + binding = _binding(client) + + verdict = await binding.register( + miner_hotkey=HOTKEY, + api_key=API_KEY, + instance_id=INSTANCE, + ) + + assert verdict.ok is False + assert verdict.reason is ConstationFailCode.POD_NOT_RUNNING + assert binding.has_binding(HOTKEY) is False + assert binding.custody.has_key(HOTKEY) is False + + +@pytest.mark.asyncio +async def test_register_mismatch_precedes_not_running() -> None: + client = ScriptedBindClient( + pod_raw=_pod_raw(miner_hotkey=OTHER, status="STOPPED"), + ) + binding = _binding(client) + + verdict = await binding.register( + miner_hotkey=HOTKEY, + api_key=API_KEY, + instance_id=INSTANCE, + ) + + assert verdict.ok is False + assert verdict.reason is ConstationFailCode.POD_HOTKEY_MISMATCH + + +# --------------------------------------------------------------------------- +# S5 get_pod transport / not-found — fail closed +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_register_pod_not_found_fail_closed() -> None: + client = ScriptedBindClient( + pod_error=LiumNotFoundError("404", status_code=404), + ) + binding = _binding(client) + + verdict = await binding.register( + miner_hotkey=HOTKEY, + api_key=API_KEY, + instance_id=INSTANCE, + ) + + assert verdict.ok is False + assert verdict.reason is ConstationFailCode.POD_HOTKEY_MISMATCH + assert binding.has_binding(HOTKEY) is False + assert binding.custody.has_key(HOTKEY) is False + + +@pytest.mark.asyncio +async def test_register_pod_401_is_lium_auth_revoked() -> None: + client = ScriptedBindClient( + pod_error=LiumAuthError("401", status_code=401), + ) + binding = _binding(client) + + verdict = await binding.register( + miner_hotkey=HOTKEY, + api_key=API_KEY, + instance_id=INSTANCE, + ) + + assert verdict.ok is False + assert verdict.reason is ConstationFailCode.LIUM_AUTH_REVOKED + assert binding.has_binding(HOTKEY) is False + assert binding.custody.has_key(HOTKEY) is False + + +@pytest.mark.asyncio +async def test_register_pod_rate_limited() -> None: + client = ScriptedBindClient( + pod_error=LiumRateLimitError("429", status_code=429), + ) + binding = _binding(client) + + verdict = await binding.register( + miner_hotkey=HOTKEY, + api_key=API_KEY, + instance_id=INSTANCE, + ) + + assert verdict.ok is False + assert verdict.reason is ConstationFailCode.LIUM_RATE_LIMITED + assert binding.has_binding(HOTKEY) is False + assert binding.custody.has_key(HOTKEY) is False + + +@pytest.mark.asyncio +async def test_register_pod_network_error_is_probe_failed() -> None: + client = ScriptedBindClient(pod_error=LiumError("network down")) + binding = _binding(client) + + verdict = await binding.register( + miner_hotkey=HOTKEY, + api_key=API_KEY, + instance_id=INSTANCE, + ) + + assert verdict.ok is False + assert verdict.reason is ConstationFailCode.PROBE_FAILED + assert binding.has_binding(HOTKEY) is False + assert binding.custody.has_key(HOTKEY) is False + + +# --------------------------------------------------------------------------- +# Blank inputs +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_register_rejects_blank_fields() -> None: + client = ScriptedBindClient() + binding = _binding(client) + + with pytest.raises(ValueError, match="miner_hotkey"): + await binding.register( + miner_hotkey=" ", + api_key=API_KEY, + instance_id=INSTANCE, + ) + with pytest.raises(ValueError, match="api_key"): + await binding.register( + miner_hotkey=HOTKEY, + api_key="", + instance_id=INSTANCE, + ) + with pytest.raises(ValueError, match="instance_id"): + await binding.register( + miner_hotkey=HOTKEY, + api_key=API_KEY, + instance_id=" ", + ) + assert binding.has_binding(HOTKEY) is False + + +# --------------------------------------------------------------------------- +# Lookup / overwrite +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_get_instance_id_missing_returns_none() -> None: + binding = _binding(ScriptedBindClient()) + assert binding.get_instance_id(HOTKEY) is None + assert binding.has_binding(HOTKEY) is False + + +@pytest.mark.asyncio +async def test_register_overwrites_prior_binding() -> None: + client = ScriptedBindClient(pod_raw=_pod_raw(pod_id="pod-a")) + binding = _binding(client) + + first = await binding.register( + miner_hotkey=HOTKEY, + api_key=API_KEY, + instance_id="pod-a", + ) + assert first.ok is True + assert binding.get_instance_id(HOTKEY) == "pod-a" + + client.pod_raw = _pod_raw(pod_id="pod-b") + second = await binding.register( + miner_hotkey=HOTKEY, + api_key=API_KEY + "-rotated", + instance_id="pod-b", + ) + assert second.ok is True + assert binding.get_instance_id(HOTKEY) == "pod-b" + assert binding.custody.unlock_api_key(HOTKEY) == API_KEY + "-rotated" diff --git a/tests/unit/test_constation_settings.py b/tests/unit/test_constation_settings.py new file mode 100644 index 00000000..370de297 --- /dev/null +++ b/tests/unit/test_constation_settings.py @@ -0,0 +1,47 @@ +"""Unit tests for ConstationSettings (production constation orchestration config).""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from base.config.loader import load_settings +from base.config.settings import ConstationSettings, Settings + + +def test_constation_settings_defaults() -> None: + settings = ConstationSettings() + assert settings.enabled is False + assert settings.gap_budget_seconds == 30.0 + assert settings.sidecar_internal_port == 8787 + assert settings.sidecar_scheme == "http" + + +def test_constation_settings_requires_positive_gap() -> None: + with pytest.raises(ValidationError): + ConstationSettings(gap_budget_seconds=0) + + +def test_constation_settings_rejects_inverted_interval() -> None: + with pytest.raises(ValidationError): + ConstationSettings(min_interval_seconds=20.0, max_interval_seconds=5.0) + + +def test_constation_settings_rejects_bad_port() -> None: + with pytest.raises(ValidationError): + ConstationSettings(sidecar_internal_port=0) + with pytest.raises(ValidationError): + ConstationSettings(sidecar_internal_port=70000) + + +def test_constation_settings_env_override(monkeypatch: pytest.MonkeyPatch) -> None: + # Convention: BASE_ prefix + nested path joined by __ (see loader._apply_env). + monkeypatch.setenv("BASE_CONSTATION__GAP_BUDGET_SECONDS", "45.5") + loaded = load_settings() + assert loaded.constation.gap_budget_seconds == 45.5 + + +def test_constation_settings_attached_to_root() -> None: + root = Settings() + assert isinstance(root.constation, ConstationSettings) + assert root.constation.enabled is False diff --git a/tests/unit/test_constation_sidecar_client.py b/tests/unit/test_constation_sidecar_client.py new file mode 100644 index 00000000..85544826 --- /dev/null +++ b/tests/unit/test_constation_sidecar_client.py @@ -0,0 +1,199 @@ +"""TDD tests for HTTP SidecarAttestor client (recipe listen POST /v1/sidecar/attest). + +Transport only: does not consume nonces. Fail-closed on malformed wire. +""" + +from __future__ import annotations + +from typing import cast + +import httpx +import pytest +import respx + +from base.compute.constation_sidecar_client import ( + ATTEST_PATH, + HttpSidecarAttestor, + SidecarAttestHit, + SidecarAttestMiss, +) +from base.compute.constation_types import ConstationFailCode + +BASE = "http://10.0.0.5:32001" +DIGEST = "sha256:" + ("ab" * 32) +NONCE = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" +POD_ID = "pod-xyz" + + +def _valid_wire( + *, + nonce: str = NONCE, + digest: str = DIGEST, + phase: str = "start", + pod_id: str = POD_ID, +) -> dict[str, object]: + return { + "schema_version": "prism_attestation_payload.v1", + "algorithm": "hmac-sha256", + "signature": "aa" * 32, + "payload": { + "nonce": nonce, + "digest": digest, + "pod_id": pod_id, + "variant": "cpu", + "sealed_manifest_hashes": {"/app/main.py": "bb" * 32}, + "build_secret_response": "cc" * 32, + }, + "phase": phase, + "hardware_root_of_trust": False, + "sufficient_alone_for_tier_elevation": False, + "proves": "entity_holding_in_image_secret_responded", + } + + +def _client(**overrides: object) -> HttpSidecarAttestor: + kwargs: dict[str, object] = {"base_url": BASE, "timeout_seconds": 2.0} + kwargs.update(overrides) + return HttpSidecarAttestor(**kwargs) # type: ignore[arg-type] + + +@respx.mock +async def test_attest_200_valid_extracts_digest() -> None: + """Given 200 valid wire; When attest; Then hit with payload digest.""" + route = respx.post(f"{BASE}{ATTEST_PATH}").mock( + return_value=httpx.Response(200, json=_valid_wire(phase="interval")) + ) + + result = await _client().attest(nonce=NONCE, phase="interval") + + assert isinstance(result, SidecarAttestHit) + assert result.digest == DIGEST + assert result.nonce == NONCE + assert result.pod_id == POD_ID + assert result.phase == "interval" + assert route.called + assert route.calls.last.request.method == "POST" + import json + + assert json.loads(route.calls.last.request.content.decode()) == { + "nonce": NONCE, + "phase": "interval", + } + + +@respx.mock +async def test_attest_400_maps_to_sidecar_response_invalid() -> None: + """Given HTTP 400; When attest; Then SIDECAR_RESPONSE_INVALID miss.""" + respx.post(f"{BASE}{ATTEST_PATH}").mock( + return_value=httpx.Response(400, json={"error": "bad request"}) + ) + + result = await _client().attest(nonce=NONCE, phase="start") + + assert isinstance(result, SidecarAttestMiss) + assert result.reason is ConstationFailCode.SIDECAR_RESPONSE_INVALID + + +@respx.mock +async def test_attest_timeout_maps_to_connection_failure() -> None: + """Given request timeout; When attest; Then typed connection/network fail.""" + respx.post(f"{BASE}{ATTEST_PATH}").mock( + side_effect=httpx.TimeoutException("timed out") + ) + + result = await _client(timeout_seconds=0.5).attest(nonce=NONCE, phase="end") + + assert isinstance(result, SidecarAttestMiss) + assert result.reason in { + ConstationFailCode.SIDECAR_ATTEST_FAILED, + ConstationFailCode.NETWORK_PARTITION, + } + + +@respx.mock +async def test_attest_missing_digest_field_fail_closed() -> None: + """Given 200 wire without payload.digest; When attest; Then invalid.""" + wire = _valid_wire() + payload = dict(cast(dict[str, object], wire["payload"])) + del payload["digest"] + wire["payload"] = payload + respx.post(f"{BASE}{ATTEST_PATH}").mock(return_value=httpx.Response(200, json=wire)) + + result = await _client().attest(nonce=NONCE, phase="start") + + assert isinstance(result, SidecarAttestMiss) + assert result.reason is ConstationFailCode.SIDECAR_RESPONSE_INVALID + + +@respx.mock +async def test_attest_blank_digest_fail_closed() -> None: + """Given 200 wire with blank digest; When attest; Then invalid.""" + respx.post(f"{BASE}{ATTEST_PATH}").mock( + return_value=httpx.Response(200, json=_valid_wire(digest=" ")) + ) + + result = await _client().attest(nonce=NONCE, phase="start") + + assert isinstance(result, SidecarAttestMiss) + assert result.reason is ConstationFailCode.SIDECAR_RESPONSE_INVALID + + +@respx.mock +async def test_attest_missing_payload_object_fail_closed() -> None: + """Given 200 JSON without payload object; When attest; Then invalid.""" + respx.post(f"{BASE}{ATTEST_PATH}").mock( + return_value=httpx.Response( + 200, + json={ + "schema_version": "prism_attestation_payload.v1", + "signature": "aa" * 32, + "digest": DIGEST, + }, + ) + ) + + result = await _client().attest(nonce=NONCE, phase="start") + + assert isinstance(result, SidecarAttestMiss) + assert result.reason is ConstationFailCode.SIDECAR_RESPONSE_INVALID + + +@respx.mock +async def test_attest_transport_error_maps_to_connection_failure() -> None: + """Given connect error; When attest; Then SIDECAR_ATTEST_FAILED.""" + respx.post(f"{BASE}{ATTEST_PATH}").mock( + side_effect=httpx.ConnectError("connection refused") + ) + + result = await _client().attest(nonce=NONCE, phase="start") + + assert isinstance(result, SidecarAttestMiss) + assert result.reason is ConstationFailCode.SIDECAR_ATTEST_FAILED + + +@respx.mock +async def test_attest_does_not_call_nonce_service() -> None: + """Given success; When attest; Then only HTTP POST (transport only).""" + respx.post(f"{BASE}{ATTEST_PATH}").mock( + return_value=httpx.Response(200, json=_valid_wire()) + ) + client = _client() + + result = await client.attest(nonce=NONCE, phase="start") + + assert isinstance(result, SidecarAttestHit) + # No nonce consume API on client + assert not hasattr(client, "consume") + assert not hasattr(client, "nonce_service") + + +def test_rejects_non_positive_timeout() -> None: + """Given timeout_seconds <= 0; When construct; Then ValueError.""" + with pytest.raises(ValueError, match="timeout"): + HttpSidecarAttestor(base_url=BASE, timeout_seconds=0.0) + + +def test_strips_trailing_slash_on_base_url() -> None: + """Given base_url with trailing slash; When construct; Then normalized.""" + client = HttpSidecarAttestor(base_url=f"{BASE}/", timeout_seconds=1.0) + assert client.base_url == BASE diff --git a/tests/unit/test_constation_sidecar_endpoint.py b/tests/unit/test_constation_sidecar_endpoint.py new file mode 100644 index 00000000..f0281e51 --- /dev/null +++ b/tests/unit/test_constation_sidecar_endpoint.py @@ -0,0 +1,176 @@ +"""TDD tests for pure Lium pod → sidecar HTTP base URL resolution. + +Fail-closed: never jupyter_url, never ssh_connect_cmd, never port guessing. +""" + +from __future__ import annotations + +from typing import Any + +from base.compute.constation_sidecar_endpoint import ( + SidecarEndpointHit, + SidecarEndpointMiss, + SidecarEndpointMissReason, + resolve_sidecar_base_url, +) + +_INTERNAL = 8787 +_HOST = "1.2.3.4" +_EXTERNAL = 32001 +_EXPECTED = f"http://{_HOST}:{_EXTERNAL}" + + +def _pod( + *, + executor_ip: object = _HOST, + ports_mapping: object = None, + extra: dict[str, Any] | None = None, +) -> dict[str, Any]: + default_mapping: dict[str, int] = {"8787": _EXTERNAL} + raw: dict[str, Any] = { + "executor": {"executor_ip_address": executor_ip}, + "ports_mapping": (default_mapping if ports_mapping is None else ports_mapping), + "status": "RUNNING", + "template": { + "internal_ports": [_INTERNAL], + "docker_image_digest": "sha256:" + ("a" * 64), + }, + } + if extra: + raw.update(extra) + return raw + + +def test_resolves_from_dict_ports_mapping() -> None: + """Given dict ports_mapping, When resolve, Then http://host:external.""" + result = resolve_sidecar_base_url(_pod(), internal_port=_INTERNAL) + + assert isinstance(result, SidecarEndpointHit) + assert result.base_url == _EXPECTED + + +def test_ports_mapping_json_string_parsed() -> None: + """Given ports_mapping as JSON string, When resolve, Then parsed and hit.""" + raw = _pod(ports_mapping='{"8787": 32001}') + + result = resolve_sidecar_base_url(raw, internal_port=_INTERNAL) + + assert isinstance(result, SidecarEndpointHit) + assert result.base_url == _EXPECTED + + +def test_ports_mapping_empty_string_default_fails() -> None: + """Given ports_mapping == '{}', When resolve, Then sidecar_port_unpublished.""" + raw = _pod(ports_mapping="{}") + + result = resolve_sidecar_base_url(raw, internal_port=_INTERNAL) + + assert isinstance(result, SidecarEndpointMiss) + assert result.reason is SidecarEndpointMissReason.SIDECAR_PORT_UNPUBLISHED + + +def test_string_and_int_keys_both_match() -> None: + """Given int key 8787, When resolve with internal_port 8787, Then hit.""" + raw = _pod(ports_mapping={8787: _EXTERNAL}) + + result = resolve_sidecar_base_url(raw, internal_port=_INTERNAL) + + assert isinstance(result, SidecarEndpointHit) + assert result.base_url == _EXPECTED + + +def test_hostport_dict_value_form() -> None: + """Given HostPort nested dict value, When resolve, Then external port extracted.""" + raw = _pod(ports_mapping={"8787": {"HostPort": "32001"}}) + + result = resolve_sidecar_base_url(raw, internal_port=_INTERNAL) + + assert isinstance(result, SidecarEndpointHit) + assert result.base_url == _EXPECTED + + +def test_hostport_list_value_form() -> None: + """Given list of HostPort dicts, When resolve, Then first valid port used.""" + raw = _pod(ports_mapping={"8787": [{"HostPort": "32001"}]}) + + result = resolve_sidecar_base_url(raw, internal_port=_INTERNAL) + + assert isinstance(result, SidecarEndpointHit) + assert result.base_url == _EXPECTED + + +def test_missing_internal_port_fails() -> None: + """Given mapping without internal_port key, When resolve, Then unpublished.""" + raw = _pod(ports_mapping={"22": 22001}) + + result = resolve_sidecar_base_url(raw, internal_port=_INTERNAL) + + assert isinstance(result, SidecarEndpointMiss) + assert result.reason is SidecarEndpointMissReason.SIDECAR_PORT_UNPUBLISHED + + +def test_missing_executor_ip_fails() -> None: + """Given blank executor_ip_address, When resolve, Then pod_endpoint_missing.""" + raw = _pod(executor_ip="") + + result = resolve_sidecar_base_url(raw, internal_port=_INTERNAL) + + assert isinstance(result, SidecarEndpointMiss) + assert result.reason is SidecarEndpointMissReason.POD_ENDPOINT_MISSING + + +def test_unparseable_ports_mapping_fails() -> None: + """Given invalid JSON ports_mapping string, When resolve, Then unparseable.""" + raw = _pod(ports_mapping="{not-json") + + result = resolve_sidecar_base_url(raw, internal_port=_INTERNAL) + + assert isinstance(result, SidecarEndpointMiss) + assert result.reason is SidecarEndpointMissReason.PORTS_MAPPING_UNPARSEABLE + + +def test_out_of_range_port_fails() -> None: + """Given external port outside 1..65535, When resolve, Then unpublished.""" + raw = _pod(ports_mapping={"8787": 70000}) + + result = resolve_sidecar_base_url(raw, internal_port=_INTERNAL) + + assert isinstance(result, SidecarEndpointMiss) + assert result.reason is SidecarEndpointMissReason.SIDECAR_PORT_UNPUBLISHED + + +def test_never_falls_back_to_jupyter_url() -> None: + """Given jupyter_url present but no mapped sidecar port, When resolve, Then fail.""" + raw = _pod( + ports_mapping={}, + extra={"jupyter_url": "https://jupyter.example/lab?token=abc"}, + ) + + result = resolve_sidecar_base_url(raw, internal_port=_INTERNAL) + + assert isinstance(result, SidecarEndpointMiss) + assert result.reason is SidecarEndpointMissReason.SIDECAR_PORT_UNPUBLISHED + + +def test_never_uses_ssh_connect_cmd() -> None: + """Given ssh_connect_cmd but no mapped sidecar port, When resolve, Then fail.""" + raw = _pod( + ports_mapping={}, + extra={"ssh_connect_cmd": f"ssh root@{_HOST} -p 22001"}, + ) + + result = resolve_sidecar_base_url(raw, internal_port=_INTERNAL) + + assert isinstance(result, SidecarEndpointMiss) + assert result.reason is SidecarEndpointMissReason.SIDECAR_PORT_UNPUBLISHED + + +def test_ipv6_host_is_bracketed() -> None: + """Given IPv6 executor_ip_address, When resolve, Then bracketed host in URL.""" + ipv6 = "2001:db8::1" + raw = _pod(executor_ip=ipv6, ports_mapping={"8787": _EXTERNAL}) + + result = resolve_sidecar_base_url(raw, internal_port=_INTERNAL) + + assert isinstance(result, SidecarEndpointHit) + assert result.base_url == f"http://[{ipv6}]:{_EXTERNAL}" diff --git a/tests/unit/test_constation_triangle.py b/tests/unit/test_constation_triangle.py new file mode 100644 index 00000000..3c7eaf38 --- /dev/null +++ b/tests/unit/test_constation_triangle.py @@ -0,0 +1,168 @@ +"""TDD tests for three-way digest triangle (required / Lium / sidecar).""" + +from __future__ import annotations + +from base.compute.constation_triangle import ( + DigestTriangleResult, + evaluate_digest_triangle, +) +from base.compute.constation_types import ( + ConstationFailCode, + FaultClass, + fault_class_for, +) + +DIGEST_A = "sha256:" + ("a" * 64) +DIGEST_B = "sha256:" + ("b" * 64) + +_NEW_FAIL_CODES: tuple[ConstationFailCode, ...] = ( + ConstationFailCode.REQUIRED_DIGEST_MISMATCH, + ConstationFailCode.LIUM_DIGEST_ABSENT, + ConstationFailCode.SIDECAR_PORT_UNPUBLISHED, + ConstationFailCode.POD_HOTKEY_MISMATCH, + ConstationFailCode.POD_NOT_RUNNING, + ConstationFailCode.SIDECAR_RESPONSE_INVALID, + ConstationFailCode.BUNDLE_INCOMPLETE, +) + +_NEW_MINER_CODES: frozenset[ConstationFailCode] = frozenset( + { + ConstationFailCode.REQUIRED_DIGEST_MISMATCH, + ConstationFailCode.LIUM_DIGEST_ABSENT, + ConstationFailCode.SIDECAR_PORT_UNPUBLISHED, + ConstationFailCode.POD_HOTKEY_MISMATCH, + ConstationFailCode.POD_NOT_RUNNING, + ConstationFailCode.SIDECAR_RESPONSE_INVALID, + } +) + + +def test_triangle_all_equal_ok() -> None: + # Given three channels report the same digest + # When evaluate_digest_triangle runs + result = evaluate_digest_triangle( + required=DIGEST_A, + lium_declared=DIGEST_A, + sidecar=DIGEST_A, + ) + # Then agreement is ok and never elevates (ok/fail only) + assert isinstance(result, DigestTriangleResult) + assert result.ok is True + assert result.fail_code is None + + +def test_triangle_mismatch_fail_closed() -> None: + # Given sidecar diverges from required + # When evaluate_digest_triangle runs + result = evaluate_digest_triangle( + required=DIGEST_A, + lium_declared=DIGEST_A, + sidecar=DIGEST_B, + ) + # Then fail closed with REQUIRED_DIGEST_MISMATCH + assert result.ok is False + assert result.fail_code is ConstationFailCode.REQUIRED_DIGEST_MISMATCH + + +def test_triangle_lium_divergence_is_corroboration_mismatch() -> None: + # Given Lium declared diverges from required (sidecar matches required) + # When evaluate_digest_triangle runs + result = evaluate_digest_triangle( + required=DIGEST_A, + lium_declared=DIGEST_B, + sidecar=DIGEST_A, + ) + # Then reuse existing CORROBORATION_MISMATCH + assert result.ok is False + assert result.fail_code is ConstationFailCode.CORROBORATION_MISMATCH + + +def test_triangle_absent_lium_fails() -> None: + # Given Lium declared digest is blank/None + # When evaluate_digest_triangle runs + result = evaluate_digest_triangle( + required=DIGEST_A, + lium_declared=None, + sidecar=DIGEST_A, + ) + # Then LIUM_DIGEST_ABSENT + assert result.ok is False + assert result.fail_code is ConstationFailCode.LIUM_DIGEST_ABSENT + + blank = evaluate_digest_triangle( + required=DIGEST_A, + lium_declared=" ", + sidecar=DIGEST_A, + ) + assert blank.ok is False + assert blank.fail_code is ConstationFailCode.LIUM_DIGEST_ABSENT + + +def test_triangle_absent_sidecar_fails() -> None: + # Given sidecar digest is blank/None + # When evaluate_digest_triangle runs + result = evaluate_digest_triangle( + required=DIGEST_A, + lium_declared=DIGEST_A, + sidecar=None, + ) + # Then SIDECAR_RESPONSE_INVALID + assert result.ok is False + assert result.fail_code is ConstationFailCode.SIDECAR_RESPONSE_INVALID + + blank = evaluate_digest_triangle( + required=DIGEST_A, + lium_declared=DIGEST_A, + sidecar=" ", + ) + assert blank.ok is False + assert blank.fail_code is ConstationFailCode.SIDECAR_RESPONSE_INVALID + + +def test_triangle_absent_required_fails() -> None: + # Given required digest is blank/None + # When evaluate_digest_triangle runs + result = evaluate_digest_triangle( + required=None, + lium_declared=DIGEST_A, + sidecar=DIGEST_A, + ) + # Then REQUIRED_DIGEST_MISMATCH (we must always know what we require) + assert result.ok is False + assert result.fail_code is ConstationFailCode.REQUIRED_DIGEST_MISMATCH + + blank = evaluate_digest_triangle( + required="", + lium_declared=DIGEST_A, + sidecar=DIGEST_A, + ) + assert blank.ok is False + assert blank.fail_code is ConstationFailCode.REQUIRED_DIGEST_MISMATCH + + +def test_triangle_case_and_whitespace_insensitive() -> None: + # Given digests differ only by case and surrounding whitespace + upper = "SHA256:" + ("AB" * 32) + lower_padded = " sha256:" + ("ab" * 32) + " " + # When evaluate_digest_triangle runs + result = evaluate_digest_triangle( + required=upper, + lium_declared=lower_padded, + sidecar=lower_padded, + ) + # Then they compare equal + assert result.ok is True + assert result.fail_code is None + + +def test_new_fail_codes_have_fault_classification() -> None: + # Given every newly added fail code + # When fault_class_for is consulted + # Then each code is classified (miner vs infra) + for code in _NEW_FAIL_CODES: + fault = fault_class_for(code) + if code is ConstationFailCode.BUNDLE_INCOMPLETE: + assert fault is FaultClass.INFRA, code + else: + assert code in _NEW_MINER_CODES + assert fault is FaultClass.MINER, code diff --git a/tests/unit/test_digest_allowlist_repository.py b/tests/unit/test_digest_allowlist_repository.py index 674d687b..60f694ce 100644 --- a/tests/unit/test_digest_allowlist_repository.py +++ b/tests/unit/test_digest_allowlist_repository.py @@ -39,12 +39,19 @@ def _record( tree_sha: str = TREE_A, variant: ImageVariant = ImageVariant.CUDA, digest: str = DIGEST_CUDA, + sealed_manifest_hashes: dict[str, str] | None = None, ) -> DigestRecord: + hashes = ( + sealed_manifest_hashes + if sealed_manifest_hashes is not None + else {"default.py": "e" * 64} + ) return DigestRecord( commit_sha=commit_sha, tree_sha=tree_sha, variant=variant, digest=digest, + sealed_manifest_hashes=hashes, ) @@ -192,3 +199,24 @@ async def test_identical_reregister_is_noop( select(func.count()).select_from(ImageDigestAllowlistEntry) ) assert count == 1 + + +@pytest.mark.asyncio +async def test_allowlist_repository_roundtrips_sealed_hashes( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Given sealed hashes, When load_allowlist, Then they round-trip.""" + sealed = {"harness.py": "a" * 64, "lib/mod.py": "b" * 64} + repo = DigestAllowlistRepository(session_factory) + record = _record(sealed_manifest_hashes=sealed) + await repo.register(record) + + allowlist = await DigestAllowlistRepository(session_factory).load_allowlist() + result = allowlist.lookup( + digest=DIGEST_CUDA, + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + ) + assert isinstance(result, AllowlistHit) + assert dict(result.record.sealed_manifest_hashes) == sealed diff --git a/tests/unit/test_e2e_lium_attestation_offline.py b/tests/unit/test_e2e_lium_attestation_offline.py index 8ad59baa..a26d8491 100644 --- a/tests/unit/test_e2e_lium_attestation_offline.py +++ b/tests/unit/test_e2e_lium_attestation_offline.py @@ -81,7 +81,7 @@ async def test_offline_adversarial_midrun_swap_no_score_miner_fault(e2e) -> None errors = e2e._assert_adversarial(bag) assert not errors, errors assert bag["run_record_ok"] is False - assert bag["run_record_reason"] == "corroboration_mismatch" + assert bag["run_record_reason"] == "required_digest_mismatch" assert bag["constation_ok"] is False assert bag["score_written"] is False assert bag["score_row"] is None diff --git a/tests/unit/test_validator_challenge_adapters.py b/tests/unit/test_validator_challenge_adapters.py index bea83b22..99bfdb79 100644 --- a/tests/unit/test_validator_challenge_adapters.py +++ b/tests/unit/test_validator_challenge_adapters.py @@ -137,7 +137,19 @@ async def _fake_dispatch(**kwargs: Any) -> dict[str, Any]: assert sent["payload"]["gateway_token"] == "scoped-token" -async def test_prism_adapter_unavailable_raises() -> None: +async def test_prism_adapter_unavailable_raises( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Monorepo workspace always has prism_challenge importable; simulate the + # lazy-import failure path the adapter documents for standalone installs. + import base.validator.agent.adapters.prism as prism_adapter + + def _boom() -> object: + raise AssignmentExecutionError( + "prism dispatch adapter is unavailable: simulated" + ) + + monkeypatch.setattr(prism_adapter, "_load_dispatch", _boom) adapter = PrismCycleExecutor() with pytest.raises(AssignmentExecutionError) as excinfo: await adapter.execute(