feat: promote production-mvp to main — drives, scan history, files modularization

cleanup_usage_event_contamination.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""One-time cleanup after the Cycle 1 Task B property-attribution fix.
+
+Before the fix, ``extract_labels_and_properties`` attributed every property it
+found to every label in the query, so
+
+    MATCH (p:Project)-[:PI_OF]-(u:Person) RETURN p.cac_protocol, u.email
+
+logged cac_protocol and email against *both* Project and Person. Those rows are
+in ``usage_event``, and ``flush_rankings()`` has already folded them into
+``property_ranking``, which shapes the chat schema context.
+
+This script deletes them so rankings rebuild cleanly from new queries.
+
+Scope — deliberately narrower than a blanket ``DELETE FROM usage_event``:
+
+  * ``event_type = 'query_executed'`` rows are deleted. These are the only rows
+    the buggy parser produced.
+  * ``event_type = 'concept_graph_plan'`` rows are KEPT. They are written
+    directly by api_chat.py with ``label_name = ''`` and
+    ``property_name = NULL``, carry the Concept Graph traversal log in
+    ``traversal_json``, and never went through the buggy parser. A blanket
+    delete would throw them away for no benefit.
+  * ``property_ranking`` is truncated. ``flush_rankings()`` upserts and never
+    deletes, so a contaminated (label, property) pair would otherwise survive in
+    the table that feeds chat context even after its source rows are gone.
+
+Idempotent — safe to run twice; the second run reports zero rows.
+
+Usage:
+    python cleanup_usage_event_contamination.py [--db PATH] [--dry-run]
+
+The database defaults to $SCIDK_SETTINGS_DB, then ./scidk_settings.db.
+"""
+import argparse
+import os
+import sqlite3
+import sys
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument(
+        '--db',
+        default=os.environ.get('SCIDK_SETTINGS_DB') or 'scidk_settings.db',
+        help='Path to scidk_settings.db (default: $SCIDK_SETTINGS_DB or ./scidk_settings.db)',
+    )
+    parser.add_argument(
+        '--dry-run', action='store_true',
+        help='Report what would be deleted without changing anything',
+    )
+    args = parser.parse_args(argv)
+
+    if not os.path.exists(args.db):
+        print(f"error: no such database: {args.db}", file=sys.stderr)
+        return 1
+
+    conn = sqlite3.connect(args.db)
+    try:
+        table = conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name='usage_event'"
+        ).fetchone()
+        if not table:
+            print(f"{args.db}: no usage_event table — nothing to clean.")
+            return 0
+
+        contaminated = conn.execute(
+            "SELECT COUNT(*) FROM usage_event WHERE event_type = 'query_executed'"
+        ).fetchone()[0]
+        kept = conn.execute(
+            "SELECT COUNT(*) FROM usage_event WHERE event_type <> 'query_executed'"
+        ).fetchone()[0]
+        rankings = conn.execute(
+            "SELECT COUNT(*) FROM property_ranking"
+        ).fetchone()[0]
+
+        print(f"{args.db}")
+        print(f"  usage_event 'query_executed' rows to delete: {contaminated}")
+        print(f"  usage_event other rows kept (traversal logs): {kept}")
+        print(f"  property_ranking rows to clear:               {rankings}")
+
+        if args.dry_run:
+            print("  (dry run — nothing changed)")
+            return 0
+
+        conn.execute("DELETE FROM usage_event WHERE event_type = 'query_executed'")
+        conn.execute("DELETE FROM property_ranking")
+        conn.commit()
+        print("  done — rankings will rebuild from new queries.")
+        return 0
+    finally:
+        conn.close()
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

docs/DEPLOYMENT.md

@@ -55,6 +55,25 @@ This guide covers production deployment of SciDK, including installation, config
    pip install -e .[dev]
    ```
 
+   **Whole-slide imaging (`imaging` extra)** — a facility holding histology
+   data should install this:
+
+   ```bash
+   pip install -e .[imaging]
+   ```
+
+   It pulls in `tifffile`, which `svs_interpreter` uses to read the
+   `ImageDescription` tag of Aperio `.svs`, Hamamatsu `.ndpi` and Leica `.scn`
+   slides (metadata only — pixel data is never loaded). Omitting it is not a
+   breaking omission: the interpreter returns a stub with a
+   `tifffile not installed` warning and enrichment continues. But no
+   `HistologySlide` or `HistologySession` nodes will appear in the graph, and
+   the omission is only visible in the `warnings` list of an enrichment run,
+   so it is easy to mistake for "there is no histology data here".
+
+   Flow cytometry (`.fcs`) needs nothing extra — `fcs_interpreter` is stdlib
+   only.
+
 4. **Initialize environment**:
    ```bash
    # bash/zsh:

docs/OPERATIONS.md

@@ -150,6 +150,38 @@ SciDK includes a backup scheduler. Configure in Settings → Backup:
 - Set retention policy
 - Configure backup location
 
+> **Schedule changes need a restart under gunicorn.** All recurring jobs run in a
+> single scheduler owned by the gunicorn master process (`--preload` in
+> `restart_gunicorn.sh`), which is what stops every job firing once per worker.
+> API requests are served by workers, so a schedule change is saved to
+> `scidk_settings.db` but does not reach the live scheduler until the app
+> restarts. A warning is logged when this happens — grep for "does not own the
+> running scheduler".
+
+### Scheduled Jobs
+
+| Job id | Default schedule | Timezone |
+|---|---|---|
+| `daily_backup` | 02:00, configurable in Settings → Backup | system-local |
+| `concept_graph_weight_decay` | 03:00 | system-local |
+| `schema_intelligence_ranking_flush` | every 6h at :17 | UTC |
+
+Environment overrides:
+
+| Variable | Default | Effect |
+|---|---|---|
+| `SCIDK_RANKING_FLUSH_INTERVAL_HOURS` | `6` | Hours between property-ranking recomputes (1–24; out-of-range falls back to the default) |
+| `SCIDK_CONCEPT_WEIGHT_HALFLIFE_DAYS` | `90` | Half-life for Concept Graph edge weight decay |
+| `SCIDK_SCHEDULER_TIMEZONE` | `UTC` | Timezone for jobs that do not state their own |
+| `SCIDK_DISABLE_SCHEDULER` | unset | Set to `1` to start no scheduler at all — for CLI commands and one-shot scripts that import the app factory |
+
+To confirm one scheduler is running rather than sixteen, check that the job
+registration line appears exactly once per restart:
+
+```bash
+grep "Scheduler running (pid=" gunicorn.log
+```
+
 **Manual SQLite backup**:
 ```bash
 # Stop the application first (important!)