Harden the CRAB backend with the DSProd production lessons

RunKit/grid_helper_tasks.py

@@ -1,9 +1,8 @@
 import law
 import luigi
 import os
-import re
 
-from .run_tools import ps_call
+from .run_tools import PsCallError, ps_call, on_batch_node
 from .grid_tools import get_voms_proxy_info
 
 
@@ -13,14 +12,29 @@ class CreateVomsProxy(law.Task):
     def __init__(self, *args, **kwargs):
         super(CreateVomsProxy, self).__init__(*args, **kwargs)
         self.proxy_path = os.getenv("X509_USER_PROXY")
-        if os.path.exists(self.proxy_path):
-            proxy_info = get_voms_proxy_info()
-            timeleft = proxy_info.get("timeleft", 0.0)
-            if timeleft < float(self.time_limit):
-                self.publish_message(
-                    f"Removing old proxy which expires in a less than {timeleft:.1f} hours."
-                )
-                self.output().remove()
+        if not self.proxy_path:
+            raise RuntimeError("CreateVomsProxy requires X509_USER_PROXY to be set")
+
+    @property
+    def on_batch_node(self):
+        return on_batch_node()
+
+    def complete(self):
+        if not os.path.exists(self.proxy_path):
+            return False
+        try:
+            timeleft = get_voms_proxy_info().get("timeleft", 0.0)
+        except PsCallError:
+            # voms-proxy-info exits non-zero on an expired or unreadable proxy
+            return False
+        if self.on_batch_node:
+            # Any valid proxy the batch system delegated will do: its remaining lifetime
+            # is not ours to police (CRAB's lives for slightly under 24 h, i.e. below the
+            # interactive renewal threshold), and voms-proxy-init cannot run unattended on
+            # a worker. Enforcing the threshold here would delete the delegated proxy,
+            # after which every remote-storage call in the job fails.
+            return True
+        return timeleft >= float(self.time_limit)
 
     def output(self):
         return law.LocalFileTarget(self.proxy_path)
@@ -42,7 +56,16 @@ def create_proxy(self, proxy_file):
         )
 
     def run(self):
+        if self.on_batch_node:
+            raise RuntimeError(
+                f"No usable voms proxy at {self.proxy_path} on a batch node, and a new "
+                "one cannot be created there. Check that the batch system delegated a "
+                "proxy."
+            )
         proxy_file = self.output()
+        if proxy_file.exists():
+            self.publish_message("Removing old proxy.")
+            proxy_file.remove()
         self.create_proxy(proxy_file)
         if not proxy_file.exists():
             raise RuntimeError("Unable to create voms proxy")

RunKit/run_tools.py

@@ -237,6 +237,11 @@ def natural_sort(l):
     return sorted(l, key=alphanum_key)
 
 
+def on_batch_node():
+    """True inside a law remote job (HTCondor or CRAB); law exports LAW_JOB_HOME there."""
+    return bool(os.getenv("LAW_JOB_HOME"))
+
+
 def check_root_file_integrity(file_name, tmp_file=None, verbose=1):
     if tmp_file is None:
         tmp_file_desc, tmp_file = tempfile.mkstemp()

docs/workflow/arguments.md

@@ -43,9 +43,11 @@ also provides built-in options for status and cleanup.
 | `--workflow crab` | — | Submit branches via CMS CRAB (WLCG). See [CRAB](crab.md). |
 
 Optional site white/black lists go in `global.yaml` under `crab:` (not CLI flags).
-Unset whitelist ⇒ all T1/T2/T3 sites. Default `--parallel-jobs` on CRAB is 5000
-(`crab.parallel_jobs`); a new CRAB task is submitted only when at least
-`crab.refill_fraction` (default 0.2) of those slots are free. `Site.storageSite`
+Unset whitelist ⇒ all T1/T2/T3 sites; blacklisted (or auto-quarantined) sites are
+cut out of the whitelist itself — see [CRAB](crab.md). Default `--parallel-jobs`
+on CRAB is 5000 (`crab.parallel_jobs`); jobs are aggregated into CRAB tasks of at
+least `crab.refill_fraction * parallel_jobs` jobs while such a wave is still
+achievable (the tail is released immediately). `Site.storageSite`
 / `Data.outLFNDirBase` are derived from `fs_default`. Memory is
 `2000 MB * n_cpus` (`crab.memory_mb_per_cpu`; CRAB / site-guaranteed default),
 capped at the CRAB client limit (5000 MB for 1 core, `2500 MB * n_cpus` otherwise).

docs/workflow/crab.md

@@ -65,10 +65,27 @@ crab:
   # whitelist: [T2_CH_CERN]   # omit to use all T1/T2/T3 sites
   # blacklist: [T2_US_MIT]
   # parallel_jobs: 5000       # default --parallel-jobs; CLI wins if set
-  # refill_fraction: 0.2      # new CRAB task only when this fraction of slots is free
+  # refill_fraction: 0.2      # minimum wave size as a fraction of parallel_jobs
+  # poll_interval: 5          # minutes between crab status polls; CLI wins if set
   # memory_mb_per_cpu: 2000   # CRAB maxMemoryMB / n_cpus
+  # auto_blacklist:           # automatic site quarantine (on by default)
+  #   enabled: true
+  # ignore_global_blacklist: false   # waive CMS's own site blacklist (not recommended)
 ```
 
+!!! note "A `crab:` block in `user_custom.yaml` replaces the `global.yaml` one wholesale"
+    The config layers are concatenated and parsed as one YAML document, so a later
+    `crab:` mapping wins as a whole — repeat the keys you want to keep.
+
+CRAB gives the **whitelist precedence** over the blacklist: a site matched by both
+lists is *kept* (the client only prints a warning). FLAF therefore removes excluded
+sites — the configured `blacklist` and the automatic quarantine alike — from the
+whitelist itself, expanding a tier glob that covers an excluded site into the concrete
+sites it matches. The expansion uses the CRIC processing-site list (cached 24 h in
+`<analysis>/data/cms_sites.json`; a stale cache is reused if CRIC is unreachable).
+Globs covering nothing excluded are passed through unchanged, so without a blacklist
+no CRIC lookup happens at all.
+
 Memory is `2000 MB * n_cpus` (override with `crab.memory_mb_per_cpu`), matching
 the CRAB default that all sites guarantee per core. Then capped at the CRAB
 client limit: 5000 MB for 1 core, `2500 MB * n_cpus` otherwise. There is no
@@ -84,10 +101,29 @@ crab checkwrite --site=T3_CH_CERNBOX --lfn=/store/user/$USER
 | Key | Meaning |
 |---|---|
 | `whitelist` | Optional. Restricts `Site.whitelist`. Default: `T1_*`, `T2_*`, `T3_*`. |
-| `blacklist` | Optional. CRAB `Site.blacklist` (applied on top of the whitelist). |
+| `blacklist` | Optional. Sites to exclude. Removed from the whitelist itself (CRAB gives the whitelist precedence, so passing them only as `Site.blacklist` would do nothing) — tier globs covering an excluded site are expanded from the CRIC processing-site list. |
 | `parallel_jobs` | Optional. Default for `--parallel-jobs` on CRAB (CLI wins). Default: `5000`. Caps how many CRAB jobs are in flight and thus the size of each CRAB task. CRAB itself refuses more than 10 000 jobs in one task. |
-| `refill_fraction` | Optional. Submit a new CRAB task only when `parallel_jobs - n_active >= refill_fraction * parallel_jobs`. Default: `0.2`. Prevents a 1-job task every time a single job finishes. |
+| `refill_fraction` | Optional. Minimum wave size, as a fraction of `parallel_jobs`. Default: `0.2`. Jobs — unsubmitted and retries alike — are held back and aggregated into one CRAB task while a full wave is still achievable, and released immediately once running + waiting can no longer fill one (the tail of a production, and any production smaller than a wave). |
 | `memory_mb_per_cpu` | Optional. CRAB `JobType.maxMemoryMB` is this times `--n-cpus`, capped at 5000 MB (1 core) or `2500 MB * n_cpus`. Default: `2000` (CRAB / site-guaranteed per-core default). |
+| `poll_interval` | Optional. Minutes between `crab status` polls (CLI `--poll-interval` wins). Default: `5`. Each poll is one multi-MB `crab status --json` per live CRAB task. |
+| `min_runtime_min` | Optional. Floor for CRAB `maxJobRuntimeMin` (bundles must be downloaded and unpacked before the payload starts). Default: `60`. |
+| `auto_blacklist` | Optional mapping (or `false`). Automatic site quarantine, on by default — see below. |
+| `ignore_global_blacklist` | Optional. Set `true` to waive CMS's own blacklist of known-broken sites (`Site.ignoreGlobalBlacklist`). Not recommended: with an open site pool it is the main protection against burning jobs at bad sites. |
+
+### Automatic site quarantine
+
+One broken worker node fails jobs in seconds, frees its slot and takes the next job, so
+a single black hole can eat a large share of a production. FLAF keeps a rolling per-site
+record of job outcomes (`<analysis>/data/crab_site_stats.json`, harvested from `crab
+status`) and keeps a site out of the *next* CRAB task — retries included — when its
+recent jobs mostly fail. The failure rate is measured over jobs *sent* to the site
+(ended + still in flight), judged against the other sites' record, so a bug of your own
+(which fails everywhere) never quarantines anything. Tune or disable it with
+`crab.auto_blacklist`; the knobs and their defaults are documented in
+`FLAF/run_tools/crab_sites.py` (`DEFAULTS`): `min_failures: 5`, `min_failure_rate: 0.5`,
+`relative_factor: 2.0`, `min_baseline_jobs: 20`, `quarantine_hours: 6`,
+`window_hours: 24`, `max_sites: 10`. The record is per analysis and advisory — deleting
+the JSON file resets it.
 
 ## Submit
 
@@ -161,3 +197,28 @@ also use `crab status -d <project_dir>` from a CMSSW environment.
 !!! note "Test small first"
     Validate with `--workflow local --branches 0 --test 1000`, then a single CRAB branch,
     before large submissions.
+
+!!! note "The CRAB client runs with its own HOME"
+    CRAB rewrites its task cache `~/.crab3` on **every** command, status polls included —
+    with `$HOME` on AFS a multi-day production dies with `PermissionError` the moment the
+    AFS token lapses. FLAF therefore runs every `crab` invocation with
+    `HOME=$TMPDIR/flaf_crab_home_<uid>` and, except for `submit`, from that directory
+    (so `crab.log` does not land in the working area). `--proxy` is always passed
+    explicitly, so nothing from the real home is needed.
+
+!!! note "An unreadable `crab status` response is ridden out"
+    `crab status` occasionally returns output law cannot parse. FLAF retries the query
+    (3x, 15 s apart), then reports that task's jobs as *pending* — with one message per
+    task naming the first lines of what crab returned — and only raises after 10
+    consecutive unreadable polls. Any query failure is ridden out this way (an expired
+    proxy or a deleted project directory included), so a genuinely dead task surfaces
+    only when the tolerance runs out — about an hour at the default cadence. While a task is degraded this way law sees no failures
+    and resubmits nothing for it; jobs at other sites and other CRAB tasks are unaffected.
+
+!!! warning "Every job reports `unknown job id`"
+    This usually means the submission itself failed and law swallowed the cause — most
+    often the CMSSW sandbox it runs `crab` in could not be built. FLAF builds that
+    sandbox eagerly before the first submission and raises an actionable error; if you
+    still see it, check that `python` on PATH resolves to a python3 (the sandbox dumps
+    its environment with bare `python`, which modern CMSSW does not ship — the flaf_env
+    provides one) and inspect `$LAW_HOME/cms/cmssw_cache`.