Confirm the enclosure permit before serving requests after boot

apps/api/src/cora/api/main.py

@@ -1019,6 +1019,10 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None]:
                         observer=enclosure_permit_observer,
                         kernel=deps,
                         name_to_id=enclosure_permit_ids,
+                        enclosure_projection_registry=enclosure_only_registry,
+                        startup_timeout_seconds=(
+                            settings.enclosure_permit_monitor_startup_timeout_seconds
+                        ),
                     ),
                     run_supervisor_lifespan(
                         deps,

apps/api/src/cora/enclosure/_monitor.py

@@ -29,6 +29,34 @@
 `reconnect_delay_seconds` then re-subscribes. A single bad observation is
 logged and skipped (the subscription survives). Cancellation (lifespan
 shutdown) propagates out of the loop.
+
+## Startup race
+
+Between process start and the monitor's first settled read, `permit_status`
+still holds whatever a prior process instance last wrote, so a `start_run` /
+`start_procedure` preflight landing in that window can pass on a stale
+`Permitted` the current process has not confirmed. `enclosure_permit_monitor_lifespan`
+narrows most of that window, in two steps, before yielding to the rest of
+boot:
+
+1. It waits, up to a bounded timeout, for every configured enclosure to
+   produce one settled observation (a real reading or the observer's own
+   `Unknown` on a dead PV). A PV that never answers does not block boot past
+   the timeout; the loop keeps retrying in the background and that one
+   enclosure's stale reading stands, gated the same as any other stale
+   reading.
+2. It then drains the enclosure projection once, because step 1 only proves
+   an `EnclosurePermitObserved` event was appended (or an append attempt was
+   swallowed on failure); `permit_status` is a denormalized read-model
+   column the `ProjectionWorker` catches up to on its own poll cadence, and
+   the preflight reads that column, never the event stream directly. Like
+   step 1, this is bounded: a slow catch-up is logged and boot proceeds
+   rather than blocking or failing.
+
+Neither step closes the window to zero: CORA can only observe that a
+settlement happened, never that no more are pending, and a PV that never
+answers leaves that one enclosure's preflight reading exactly as stale as
+before this fix.
 """
 
 from __future__ import annotations
@@ -53,6 +81,7 @@
 from cora.enclosure.ports.enclosure_observer import EnclosureObserverScope
 from cora.infrastructure.event_envelope import to_new_event
 from cora.infrastructure.logging import get_logger
+from cora.infrastructure.projection import ProjectionDrainTimeoutError, drain_projections
 from cora.infrastructure.routing import SYSTEM_PRINCIPAL_ID
 from cora.shared.identity import MonitorSourceId
 
@@ -64,10 +93,18 @@
         EnclosureObserver,
     )
     from cora.infrastructure.kernel import Kernel
+    from cora.infrastructure.projection import ProjectionRegistry
 
 _STREAM_TYPE = "Enclosure"
 _COMMAND_NAME = "ObserveEnclosureStatus"
 _RECONNECT_DELAY_SECONDS = 5.0
+# A few seconds above EpicsCaControlPort's own _DEFAULT_TIMEOUT_S (5.0s):
+# a dead PV's per-code settlement (pump reaching Unknown via a bounded
+# connect attempt) needs room to finish before this outer bound gives up,
+# or every dead-PV boot falls through to the blunter warn-and-proceed path
+# instead of the cleaner "every code settled, including Unknown" one.
+_STARTUP_TIMEOUT_SECONDS = 8.0
+_PROJECTION_DRAIN_DEADLINE_SECONDS = 5.0
 
 # Stable monitor-source id for the enclosure permit monitor; stamped onto
 # EnclosurePermitObserved.triggered_by as the in-process adapter attribution.
@@ -152,11 +189,22 @@ async def run_enclosure_permit_monitor(
     kernel: Kernel,
     name_to_id: Mapping[str, UUID],
     reconnect_delay_seconds: float = _RECONNECT_DELAY_SECONDS,
+    startup_ready: asyncio.Event | None = None,
 ) -> None:
-    """Drain the observer, recording each observation; re-subscribe on stream end."""
+    """Drain the observer, recording each observation; re-subscribe on stream end.
+
+    `startup_ready`, when given, is set as soon as every configured enclosure
+    code has produced one observation (a real reading or the observer's own
+    `Unknown` on a dead PV), or, failing that, once the current pass ends (the
+    observer's stream terminated or raised with codes still unheard from). The
+    second case fires on every reconnect attempt, not only the first, but
+    `Event.set` on an already-set event is a no-op, so the caller only ever
+    sees the earliest settlement.
+    """
     if not name_to_id:
         return
     scope = EnclosureObserverScope(enclosure_codes=frozenset(name_to_id))
+    pending_codes: set[str] = set(name_to_id) if startup_ready is not None else set()
     while True:
         try:
             async for observation in observer.observe(scope):
@@ -169,10 +217,15 @@ async def run_enclosure_permit_monitor(
                         "enclosure_monitor.record_failed",
                         enclosure_code=observation.enclosure_code,
                     )
+                pending_codes.discard(observation.enclosure_code)
+                if startup_ready is not None and not pending_codes:
+                    startup_ready.set()
         except asyncio.CancelledError:
             raise
         except Exception:
             _log.exception("enclosure_monitor.iteration_failed")
+        if startup_ready is not None:
+            startup_ready.set()
         await asyncio.sleep(reconnect_delay_seconds)
 
 
@@ -182,20 +235,68 @@ async def enclosure_permit_monitor_lifespan(
     observer: EnclosureObserver,
     kernel: Kernel,
     name_to_id: Mapping[str, UUID],
+    enclosure_projection_registry: ProjectionRegistry | None = None,
+    startup_timeout_seconds: float = _STARTUP_TIMEOUT_SECONDS,
 ) -> AsyncGenerator[None]:
     """Run the permit monitor as a background task for the app's lifetime.
 
     No-op when `name_to_id` is empty (no enclosures configured): yields
     immediately without starting a task. Mirrors `projection_worker_lifespan`.
+
+    Otherwise, before yielding: waits up to `startup_timeout_seconds` for
+    every configured enclosure's first settled observation, then, when
+    `enclosure_projection_registry` is given and `kernel.pool` is a real
+    pool, drains it once. Both steps exist because neither alone gets a
+    preflight landing right after boot to a status this process has
+    confirmed (see module docstring, "Startup race"): the wait only proves
+    an event was appended (or an append attempt failed and was logged); the
+    drain is what catches `permit_status` itself up to that event. A PV
+    that never settles does not delay boot past `startup_timeout_seconds`:
+    the background task keeps retrying and that enclosure's existing
+    reading stands, gated the same as any other stale reading.
+    `enclosure_projection_registry` is optional so tests that stub `kernel`
+    can omit it and skip the drain entirely.
     """
     if not name_to_id:
         yield
         return
+    startup_ready = asyncio.Event()
     task = asyncio.create_task(
-        run_enclosure_permit_monitor(observer=observer, kernel=kernel, name_to_id=name_to_id),
+        run_enclosure_permit_monitor(
+            observer=observer,
+            kernel=kernel,
+            name_to_id=name_to_id,
+            startup_ready=startup_ready,
+        ),
         name="enclosure-permit-monitor",
     )
     try:
+        try:
+            await asyncio.wait_for(startup_ready.wait(), timeout=startup_timeout_seconds)
+        except TimeoutError:
+            _log.warning(
+                "enclosure_monitor.startup_timeout",
+                startup_timeout_seconds=startup_timeout_seconds,
+                enclosure_codes=sorted(name_to_id),
+            )
+        if enclosure_projection_registry is not None and kernel.pool is not None:
+            try:
+                await drain_projections(
+                    kernel.pool,
+                    enclosure_projection_registry,
+                    deadline_seconds=_PROJECTION_DRAIN_DEADLINE_SECONDS,
+                )
+            except ProjectionDrainTimeoutError:
+                # Same posture as the startup_timeout_seconds branch above:
+                # a slow catch-up degrades permit_status freshness, it must
+                # never abort boot. Letting this propagate would turn a
+                # read-honesty fix into an availability regression, which is
+                # a worse trade than the staleness this fix exists to narrow.
+                _log.warning(
+                    "enclosure_monitor.projection_drain_timeout",
+                    deadline_seconds=_PROJECTION_DRAIN_DEADLINE_SECONDS,
+                    enclosure_codes=sorted(name_to_id),
+                )
         yield
     finally:
         task.cancel()

apps/api/src/cora/infrastructure/config.py

@@ -626,6 +626,23 @@ class Settings(BaseSettings):
     # `cora.enclosure.adapters.control_port_enclosure_observer`.
     enclosure_permit_pvs: dict[str, str] = {}
 
+    # Bounds how long boot waits for the permit monitor's first settled read
+    # per configured enclosure before serving requests (the startup-race
+    # window documented in `cora.enclosure._monitor`, "Startup race"). A PV
+    # that has not settled by the deadline does not delay boot further: the
+    # monitor keeps retrying in the background and that enclosure's existing
+    # `permit_status` stands. Irrelevant when `enclosure_permit_pvs` is empty.
+    #
+    # `main.py` always passes this value explicitly, so it is the LIVE
+    # default; `_monitor._STARTUP_TIMEOUT_SECONDS` only covers a caller that
+    # skips Settings entirely. Keep both above EpicsCaControlPort's own
+    # `_DEFAULT_TIMEOUT_S` (5.0s): a dead PV's per-code settlement needs
+    # room to finish before this outer bound gives up, or a dead-PV boot
+    # always falls through to the blunter warn-and-proceed path. These two
+    # defaults drifted out of sync once already; if you change one, change
+    # both.
+    enclosure_permit_monitor_startup_timeout_seconds: float = 8.0
+
     # Beam-availability pre-flight (BEAM-1, beam-availability slice).
     # Role -> read-only PV for the run / procedure start gate. `fes` and
     # `sbs` are the front-end and station-shutter BeamBlockingM PVs