diff --git a/apps/api/src/cora/data/wire.py b/apps/api/src/cora/data/wire.py index 0c0a0e3b66..019cde5def 100644 --- a/apps/api/src/cora/data/wire.py +++ b/apps/api/src/cora/data/wire.py @@ -130,7 +130,10 @@ def _build_checksum_verifiers(deps: Kernel) -> Mapping[str, ChecksumVerifier]: verifiers: dict[str, ChecksumVerifier] = {"http": http, "https": http} roots = deps.settings.posix_checksum_roots if roots: - verifiers["file"] = PosixChecksumAdapter(allowed_roots=roots) + verifiers["file"] = PosixChecksumAdapter( + allowed_roots=roots, + max_walk_seconds=deps.settings.posix_checksum_max_walk_seconds, + ) return verifiers @@ -320,7 +323,8 @@ def wire_data(deps: Kernel) -> DataHandlers: captured_at_source=deps.settings.scan_captured_at_source, ), checksum_computer=PosixChecksumAdapter( - allowed_roots=deps.settings.posix_checksum_roots + allowed_roots=deps.settings.posix_checksum_roots, + max_walk_seconds=deps.settings.posix_checksum_max_walk_seconds, ), dataset_by_checksum_lookup=_build_dataset_by_checksum_lookup(deps), ), diff --git a/apps/api/src/cora/infrastructure/config.py b/apps/api/src/cora/infrastructure/config.py index 32da2dcf89..c62be7d08a 100644 --- a/apps/api/src/cora/infrastructure/config.py +++ b/apps/api/src/cora/infrastructure/config.py @@ -510,6 +510,20 @@ class Settings(BaseSettings): # See `cora.data.adapters.posix_checksum`. posix_checksum_roots: tuple[str, ...] = () + # End-to-end budget for one POSIX digest walk, seconds. The bound + # exists so a file on a hung mount, or one still growing, cannot + # occupy a worker indefinitely; it is not a performance knob. + # + # The 60 s default suits the small files the adapter was written + # against and is far too short for a tomography scan. Measured on + # the 2-BM pilot: `sha256sum` alone takes 82 s on a 24.5 GB scan + # (77 s of it CPU), and CORA's chunked read is slower still, so the + # first real ingest refused with `walk exceeded max_walk_seconds`. + # A deployment holding files of that size raises this to something + # that bounds a hang without forbidding its own data. Read from + # POSIX_CHECKSUM_MAX_WALK_SECONDS. + posix_checksum_max_walk_seconds: float = 60.0 + # Data BC -- which of a scan file's timestamps is the acquisition # time. `start_date` (the default) preserves the behaviour every # deployment had before this setting existed. diff --git a/apps/api/tests/unit/data/test_checksum_verifier_wiring.py b/apps/api/tests/unit/data/test_checksum_verifier_wiring.py index bba7a83294..9aa2e946c5 100644 --- a/apps/api/tests/unit/data/test_checksum_verifier_wiring.py +++ b/apps/api/tests/unit/data/test_checksum_verifier_wiring.py @@ -21,9 +21,20 @@ _ID = UUID("01900000-0000-7000-8000-0000000000c0") -def _wired_verifiers(roots: tuple[str, ...]) -> dict[str, object]: +def _wired_verifiers( + roots: tuple[str, ...], max_walk_seconds: float | None = None +) -> dict[str, object]: deps = build_deps(ids=[_ID], now=_NOW) - deps = replace(deps, settings=Settings(app_env="test", posix_checksum_roots=roots)) + settings = ( + Settings(app_env="test", posix_checksum_roots=roots) + if max_walk_seconds is None + else Settings( + app_env="test", + posix_checksum_roots=roots, + posix_checksum_max_walk_seconds=max_walk_seconds, + ) + ) + deps = replace(deps, settings=settings) wire_data(deps) # attaches deps.data.checksum_verifiers return dict(deps.data.checksum_verifiers) # type: ignore[attr-defined] @@ -46,3 +57,28 @@ def test_file_scheme_absent_when_no_roots_configured() -> None: def test_file_scheme_wired_to_posix_adapter_when_roots_configured() -> None: verifiers = _wired_verifiers(("/gpfs/2bm/archive",)) assert isinstance(verifiers["file"], PosixChecksumAdapter) + + +@pytest.mark.unit +def test_posix_adapter_takes_the_configured_walk_budget() -> None: + """The bound has to be reachable from configuration. + + It was tunable at construction time and nothing passed it, so the + 60 s default governed every deployment. A 24.5 GB scan takes 82 s to + hash on the 2-BM pilot before CORA's own chunked read is counted, so + the first real ingest refused with `walk exceeded max_walk_seconds` + and no deployment could raise it without editing code. + """ + verifiers = _wired_verifiers(("/gpfs/2bm/archive",), 900.0) + + adapter = verifiers["file"] + assert isinstance(adapter, PosixChecksumAdapter) + assert adapter._max_walk_seconds == 900.0 # pyright: ignore[reportPrivateUsage] + + +@pytest.mark.unit +def test_posix_adapter_keeps_the_default_walk_budget_when_unset() -> None: + adapter = _wired_verifiers(("/gpfs/2bm/archive",))["file"] + + assert isinstance(adapter, PosixChecksumAdapter) + assert adapter._max_walk_seconds == 60.0 # pyright: ignore[reportPrivateUsage]