Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions apps/api/src/cora/data/wire.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,10 @@ def _build_checksum_verifiers(deps: Kernel) -> Mapping[str, ChecksumVerifier]:
verifiers: dict[str, ChecksumVerifier] = {"http": http, "https": http}
roots = deps.settings.posix_checksum_roots
if roots:
verifiers["file"] = PosixChecksumAdapter(allowed_roots=roots)
verifiers["file"] = PosixChecksumAdapter(
allowed_roots=roots,
max_walk_seconds=deps.settings.posix_checksum_max_walk_seconds,
)
return verifiers


Expand Down Expand Up @@ -320,7 +323,8 @@ def wire_data(deps: Kernel) -> DataHandlers:
captured_at_source=deps.settings.scan_captured_at_source,
),
checksum_computer=PosixChecksumAdapter(
allowed_roots=deps.settings.posix_checksum_roots
allowed_roots=deps.settings.posix_checksum_roots,
max_walk_seconds=deps.settings.posix_checksum_max_walk_seconds,
),
dataset_by_checksum_lookup=_build_dataset_by_checksum_lookup(deps),
),
Expand Down
14 changes: 14 additions & 0 deletions apps/api/src/cora/infrastructure/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -510,6 +510,20 @@ class Settings(BaseSettings):
# See `cora.data.adapters.posix_checksum`.
posix_checksum_roots: tuple[str, ...] = ()

# End-to-end budget for one POSIX digest walk, seconds. The bound
# exists so a file on a hung mount, or one still growing, cannot
# occupy a worker indefinitely; it is not a performance knob.
#
# The 60 s default suits the small files the adapter was written
# against and is far too short for a tomography scan. Measured on
# the 2-BM pilot: `sha256sum` alone takes 82 s on a 24.5 GB scan
# (77 s of it CPU), and CORA's chunked read is slower still, so the
# first real ingest refused with `walk exceeded max_walk_seconds`.
# A deployment holding files of that size raises this to something
# that bounds a hang without forbidding its own data. Read from
# POSIX_CHECKSUM_MAX_WALK_SECONDS.
posix_checksum_max_walk_seconds: float = 60.0

# Data BC -- which of a scan file's timestamps is the acquisition
# time. `start_date` (the default) preserves the behaviour every
# deployment had before this setting existed.
Expand Down
40 changes: 38 additions & 2 deletions apps/api/tests/unit/data/test_checksum_verifier_wiring.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,20 @@
_ID = UUID("01900000-0000-7000-8000-0000000000c0")


def _wired_verifiers(roots: tuple[str, ...]) -> dict[str, object]:
def _wired_verifiers(
roots: tuple[str, ...], max_walk_seconds: float | None = None
) -> dict[str, object]:
deps = build_deps(ids=[_ID], now=_NOW)
deps = replace(deps, settings=Settings(app_env="test", posix_checksum_roots=roots))
settings = (
Settings(app_env="test", posix_checksum_roots=roots)
if max_walk_seconds is None
else Settings(
app_env="test",
posix_checksum_roots=roots,
posix_checksum_max_walk_seconds=max_walk_seconds,
)
)
deps = replace(deps, settings=settings)
wire_data(deps) # attaches deps.data.checksum_verifiers
return dict(deps.data.checksum_verifiers) # type: ignore[attr-defined]

Expand All @@ -46,3 +57,28 @@ def test_file_scheme_absent_when_no_roots_configured() -> None:
def test_file_scheme_wired_to_posix_adapter_when_roots_configured() -> None:
verifiers = _wired_verifiers(("/gpfs/2bm/archive",))
assert isinstance(verifiers["file"], PosixChecksumAdapter)


@pytest.mark.unit
def test_posix_adapter_takes_the_configured_walk_budget() -> None:
"""The bound has to be reachable from configuration.

It was tunable at construction time and nothing passed it, so the
60 s default governed every deployment. A 24.5 GB scan takes 82 s to
hash on the 2-BM pilot before CORA's own chunked read is counted, so
the first real ingest refused with `walk exceeded max_walk_seconds`
and no deployment could raise it without editing code.
"""
verifiers = _wired_verifiers(("/gpfs/2bm/archive",), 900.0)

adapter = verifiers["file"]
assert isinstance(adapter, PosixChecksumAdapter)
assert adapter._max_walk_seconds == 900.0 # pyright: ignore[reportPrivateUsage]


@pytest.mark.unit
def test_posix_adapter_keeps_the_default_walk_budget_when_unset() -> None:
adapter = _wired_verifiers(("/gpfs/2bm/archive",))["file"]

assert isinstance(adapter, PosixChecksumAdapter)
assert adapter._max_walk_seconds == 60.0 # pyright: ignore[reportPrivateUsage]
Loading