From 88aed766bdf23f79eeee26a6bf65d435466ca131 Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Sun, 16 Aug 2026 07:46:07 -0400 Subject: [PATCH 01/15] docs: specify split-screen Phase 2, and correct stale edgeFile notes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Phase 2 plan existed only in conversation and was lost to a context compaction, leaving two comments in store.js pointing at a design nobody could read. This writes it down as a specification. The substance: panel *settings* become per-panel with a global link toggle defaulting to linked, rather than global-plus-overrides (two sources of truth for every value, and no clean answer for what a slider shows) or always-independent (breaks the default case and reintroduces the drift linkColorScale exists to prevent). The push-to-other-panel button that prompted this work falls out of per-panel state as an object copy. Surface is measured rather than guessed: 4 files, 17 bare useStore() destructure sites, 25 selector calls, 19 LayerPanel sections — and notably 0 data hooks, since they take settings as props from ViewerPanel rather than reading the store. Staged 2a–2e so the large mechanical migration lands while behaviour is still frozen and any regression is unambiguous. Also recorded, both found while checking the plan against the code: - CLAUDE.md described edgeFile as global in three places. Phase 1 moved it to panels[i].edgeFile and left no global behind, so the docs contradicted the code. Corrected here rather than deferred — a stale architecture note is worse than a missing one. - regions and measurements carry no panelIndex and Viewer renders them unfiltered, so every region draws in both panels at identical image-pixel coordinates. Harmless with one dataset in both panels, wrong with two: a polygon on a 6.5 mm Visium capture area redraws at those coordinates on a 55 µm seqFISH ROI. Logged as a Phase 1 bug to fix separately, not folded into this plan. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 33 +++--- docs/split_screen_phase2.md | 205 ++++++++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+), 14 deletions(-) create mode 100644 docs/split_screen_phase2.md diff --git a/CLAUDE.md b/CLAUDE.md index e8d1559..cfde85d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -203,6 +203,9 @@ docs/ cloud-deploy.md DigitalOcean deployment runbook (~$106–116/mo) public_datasets.md Public datasets used for development, and the classic Visium pair the reader was verified against + split_screen_phase2.md Spec for making panel *settings* per-panel. Phase 1 + (per-panel datasets) shipped in v0.8.4; this is what + remains. Read before touching store.js's shared state. index.html The user manual, published to GitHub Pages at https://raredonlab.github.io/TissuePlex/ — hand-written HTML, no build step. Update it when a UI control changes. @@ -368,12 +371,14 @@ param (e.g. `"edges/edge.raw.minimum.parquet"`); `label` is the display name `(dataset, edge_file)`. `_reader()` resolves `edge_file` under the dataset directory and rejects anything that escapes it (path-traversal guard → 400; missing file → 404). -**Frontend** — a single global `edgeFile` in the store applies to **all** open viewer -panels (see the Split-Screen note; a per-panel edge file was deliberately deferred -because LRM catalogue / color ranges are edge-file-specific and the sidebar is shared). -The picker is a ` setLinkSettings(e.target.checked)} + /> + + {linkSettings + ? "settings linked — edits apply to both panels" + : `editing panel ${activePanel + 1} only`} + + + + ); +} + function LinkColorScaleRow() { const { linkColorScale, setLinkColorScale } = usePanelSettings(); return ( diff --git a/frontend/src/store.js b/frontend/src/store.js index b1f58ac..cb6eef6 100644 --- a/frontend/src/store.js +++ b/frontend/src/store.js @@ -63,6 +63,25 @@ export function makeSettings() { }; } +/** + * Copy a settings object deeply enough that the panels cannot alias. + * + * Only the mutable containers need rebuilding — every setter replaces rather + * than mutates, so the leaves can be shared. + */ +function cloneSettings(src) { + return { + ...src, + hiddenLrms: new Set(src.hiddenLrms), + selectedGenes: src.selectedGenes === null ? null : new Set(src.selectedGenes), + layers: Object.fromEntries( + Object.entries(src.layers).map(([k, v]) => [k, { ...v }])), + categoricalOverrides: { ...src.categoricalOverrides }, + categoryColorOverrides: { ...src.categoryColorOverrides }, + transcriptColorOverrides: { ...src.transcriptColorOverrides }, + }; +} + /** * One panel's dataset-bound state. * @@ -138,14 +157,20 @@ export const useStore = create((set, get) => ({ cellColorClamp: { low: null, high: null }, edgeColorClamp: { low: null, high: null }, }; - // Applied to EVERY panel, which is the pre-existing behaviour and its - // pre-existing cost: switching one panel's dataset clears the other panel's - // filter. Phase 2a is deliberately behaviour-frozen, so that stays for now. - // Narrowing this to panel `i` is Phase 2d and is a one-word change here - // (`idx === i ? {...} : p.settings`) once the link toggle exists. + // The panel whose dataset changed is always reset. The *others* are reset + // only while the panels are linked, where they share one set of values and + // leaving them would strand a filter naming a column the new dataset lacks + // — which 400s on every viewport change. + // + // Unlinked, reaching across would contradict the toggle the user just set: + // the sidebar says "editing panel 1 only" while an action on panel 2 + // destroys panel 1's work. That was the pre-2b behaviour and the cost + // recorded in CLAUDE.md; the link toggle is what makes it fixable, so 2d + // lands with 2b rather than after it. const panels = s.panels.map((p, idx) => { const base = idx === i ? { ...makePanel(), dataset } : p; - return { ...base, settings: { ...p.settings, ...RESET } }; + const reset = idx === i || s.linkSettings; + return { ...base, settings: reset ? { ...p.settings, ...RESET } : p.settings }; }); // Selections belong to a dataset; drop any that pointed at the old one. const sel = s.selection && s.selection.panelIndex === i ? null : s.selection; @@ -162,11 +187,11 @@ export const useStore = create((set, get) => ({ }; const sel = s.selection && s.selection.panelIndex === i && s.selection.kind === "edge" ? null : s.selection; - // Mechanism and edge-column names are edge-file specific; cleared on every - // panel, matching the pre-existing global behaviour. - const panels = next.map((p) => ({ - ...p, settings: { ...p.settings, hiddenLrms: new Set(), edgeFilter: null }, - })); + // Mechanism and edge-column names are specific to one edge file, so the + // panel that changed is always cleared; the others only while linked. + const panels = next.map((p, idx) => (idx === i || s.linkSettings) + ? { ...p, settings: { ...p.settings, hiddenLrms: new Set(), edgeFilter: null } } + : p); return { panels, selection: sel }; }), @@ -261,29 +286,74 @@ export const useStore = create((set, get) => ({ // function becomes the single place where "write to one panel or all of them" // is decided. Keeping the named setters means call sites never had to change. - /** Merge a patch into one panel's settings, or every panel's. */ - patchSettings: (patch, panelIndex = null) => - set((s) => ({ - panels: s.panels.map((p, i) => - panelIndex === null || i === panelIndex - ? { ...p, settings: { ...p.settings, ...patch } } - : p), - })), + // ── Which panel the sidebar edits, and whether edits propagate ──────────── + // + // activePanel is the tab the sidebar is pointed at. It is also what the + // sidebar *reads*, so with the panels unlinked the controls show the values + // for the panel you are editing rather than some blend. + // + // linkSettings defaults to true, which reproduces the pre-2b behaviour + // exactly: one sidebar drives both panels. That default is deliberate — two + // panels that each drifted to their own palette and clamp look comparable and + // are not, which is the same figure-integrity argument behind linkColorScale. + activePanel: 0, + setActivePanel: (i) => set({ activePanel: i }), + + linkSettings: true, + // Re-linking *syncs*: every panel adopts the active panel's settings. The + // alternative — start propagating future edits but leave the existing + // divergence in place — leaves a control labelled "linked" over two panels + // that visibly differ, and the next single edit converges them only + // partially. Adopting one panel's state is the only reading of "linked" that + // is true the moment you switch it on. Which panel wins is the tab you are + // on, so it is visible and chosen rather than incidental. + setLinkSettings: (v) => set((s) => { + if (!v) return { linkSettings: false }; + const source = s.panels[s.activePanel]?.settings ?? s.panels[0].settings; + return { + linkSettings: true, + panels: s.panels.map((p) => ({ ...p, settings: cloneSettings(source) })), + }; + }), - /** Read the effective value of one setting. Panel 0 is the reference while - * settings are still written to every panel in lockstep. */ - getSetting: (key, panelIndex = 0) => get().panels[panelIndex]?.settings?.[key], + /** + * Merge a patch into panel settings. + * + * An explicit `panelIndex` always wins. Otherwise the link state decides: all + * panels when linked, just the active one when not. This is the single place + * that choice is made — every named setter routes through here, so none of + * them has to know about tabs or linking. + */ + patchSettings: (patch, panelIndex = null) => + set((s) => { + const targets = panelIndex !== null + ? [panelIndex] + : (s.linkSettings ? s.panels.map((_, i) => i) : [s.activePanel]); + return { + panels: s.panels.map((p, i) => + targets.includes(i) + ? { ...p, settings: { ...p.settings, ...patch } } + : p), + }; + }), + + /** + * Read one setting. Defaults to the panel the sidebar is editing, which is + * what read-modify-write setters need: unlinked, `toggleLrm` must toggle + * against the active panel's Set, not panel 0's. + */ + getSetting: (key, panelIndex = null) => { + const s = get(); + const i = panelIndex ?? s.activePanel; + return s.panels[i]?.settings?.[key]; + }, - setLayerProp: (id, prop, value) => - set((s) => ({ - panels: s.panels.map((p) => ({ - ...p, - settings: { - ...p.settings, - layers: { ...p.settings.layers, [id]: { ...p.settings.layers[id], [prop]: value } }, - }, - })), - })), + // Read-modify-write on a nested map, so it derives from the active panel and + // then goes through the ordinary targeting rule. + setLayerProp: (id, prop, value) => { + const cur = get().getSetting("layers"); + get().patchSettings({ layers: { ...cur, [id]: { ...cur[id], [prop]: value } } }); + }, setCellBoundaryFraction: (v) => get().patchSettings({ cellBoundaryFraction: v !== null ? Math.max(0.0001, Math.min(1.0, v)) : null, diff --git a/frontend/src/store.settings.test.js b/frontend/src/store.settings.test.js index 6a72463..4057c8c 100644 --- a/frontend/src/store.settings.test.js +++ b/frontend/src/store.settings.test.js @@ -19,9 +19,16 @@ const S = () => useStore.getState(); const settings = (i) => S().panels[i].settings; // Built through the store's own factory so these tests cannot drift from the -// real defaults. +// real defaults. activePanel and linkSettings are reset too — they are what the +// write-targeting rule reads, so leaving them set by a previous test silently +// redirects the next one's writes to the wrong panel. beforeEach(() => { - useStore.setState({ panels: [makePanel(), makePanel()], panelCount: 2 }); + useStore.setState({ + panels: [makePanel(), makePanel()], + panelCount: 2, + activePanel: 0, + linkSettings: true, + }); }); describe("settings live on the panel, not the store root", () => { @@ -124,13 +131,31 @@ describe("a dataset change resets only the name-bound settings", () => { expect(settings(0).layers.transcripts.visible).toBe(true); }); - it("still resets the other panel too, as it did before 2a", () => { - // The known cost documented in CLAUDE.md. Phase 2d narrows this to the one - // panel; until then, changing it would be an unplanned behaviour change. + it("resets the other panel too while the panels are linked", () => { + // Linked panels share one set of values, so leaving the other alone would + // strand a filter naming a column the new dataset lacks. + expect(S().linkSettings).toBe(true); S().setPanelDataset(0, "other-dataset"); expect(settings(1).cellFilter).toBeNull(); expect(settings(1).colorBy).toEqual({ mode: "off", field: null }); }); + + it("leaves the other panel alone once unlinked (2d)", () => { + // The toggle says "editing panel 1 only"; an action on panel 2 must not + // destroy panel 1's work. This was the cost recorded in CLAUDE.md, and the + // link toggle is what makes it fixable. + S().setLinkSettings(false); + S().setActivePanel(0); + S().setCellFilter({ field: "cluster", values: ["4"] }); + S().setColorBy("metadata", "cluster"); + + S().setPanelDataset(1, "other-dataset"); + expect(settings(0).cellFilter).toEqual({ field: "cluster", values: ["4"] }); + expect(settings(0).colorBy).toEqual({ mode: "metadata", field: "cluster" }); + // ...while the panel that actually changed is still cleared. + expect(settings(1).cellFilter).toBeNull(); + expect(settings(1).colorBy).toEqual({ mode: "off", field: null }); + }); }); describe("gene toggle reads and writes through settings", () => { @@ -153,3 +178,96 @@ describe("gene toggle reads and writes through settings", () => { expect(settings(0).selectedGenes).toBeNull(); }); }); + +describe("2b — the link toggle decides where a write lands", () => { + it("linked (the default) still writes to every panel", () => { + expect(S().linkSettings).toBe(true); + S().setEdgeWidth(7); + expect(settings(0).edgeWidth).toBe(7); + expect(settings(1).edgeWidth).toBe(7); + }); + + it("unlinked writes only to the active panel", () => { + S().setLinkSettings(false); + S().setActivePanel(1); + S().setEdgeWidth(7); + expect(settings(0).edgeWidth).toBe(2); + expect(settings(1).edgeWidth).toBe(7); + }); + + it("unlinked, a nested layer edit stays in its panel", () => { + S().setLinkSettings(false); + S().setActivePanel(1); + S().setLayerProp("transcripts", "visible", true); + expect(settings(0).layers.transcripts.visible).toBe(false); + expect(settings(1).layers.transcripts.visible).toBe(true); + }); + + it("unlinked, a read-modify-write setter reads the panel it writes", () => { + // toggleLrm builds the next Set from the current one. Reading panel 0 while + // writing panel 1 would drop whatever panel 1 already had hidden. + S().setLinkSettings(false); + S().setActivePanel(1); + S().toggleLrm("A|B"); + S().toggleLrm("C|D"); + expect([...settings(1).hiddenLrms].sort()).toEqual(["A|B", "C|D"]); + expect([...settings(0).hiddenLrms]).toEqual([]); + }); + + it("switching tabs does not itself change anything", () => { + S().setLinkSettings(false); + S().setEdgeWidth(5); // panel 0 + S().setActivePanel(1); + expect(settings(0).edgeWidth).toBe(5); + expect(settings(1).edgeWidth).toBe(2); + }); +}); + +describe("2b — re-linking adopts the active panel's settings", () => { + beforeEach(() => { + S().setLinkSettings(false); + S().setActivePanel(0); + S().setEdgeWidth(3); + S().setActivePanel(1); + S().setEdgeWidth(9); + S().setCellColorPalette("plasma"); + }); + + it("copies the tab you are on onto the others", () => { + S().setActivePanel(1); + S().setLinkSettings(true); + expect(settings(0).edgeWidth).toBe(9); + expect(settings(1).edgeWidth).toBe(9); + expect(settings(0).cellColorPalette).toBe("plasma"); + }); + + it("the other tab wins if that is the one you are on", () => { + S().setActivePanel(0); + S().setLinkSettings(true); + expect(settings(0).edgeWidth).toBe(3); + expect(settings(1).edgeWidth).toBe(3); + }); + + it("does not leave the panels sharing containers", () => { + // A shallow copy on re-link would alias the layer maps and Sets, so the + // next unlink-and-edit would write through to both panels. + S().setActivePanel(1); + S().setLinkSettings(true); + expect(settings(0).layers).not.toBe(settings(1).layers); + expect(settings(0).hiddenLrms).not.toBe(settings(1).hiddenLrms); + + S().setLinkSettings(false); + S().setActivePanel(1); + S().setLayerProp("edges", "visible", true); + expect(settings(0).layers.edges.visible).toBe(false); + }); + + it("unlinking on its own changes no values", () => { + S().setActivePanel(1); + S().setLinkSettings(true); + const before = settings(0).edgeWidth; + S().setLinkSettings(false); + expect(settings(0).edgeWidth).toBe(before); + expect(settings(1).edgeWidth).toBe(before); + }); +}); From bae4fd73f2d883f46298dd11627e18e548e65a9e Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Sun, 16 Aug 2026 08:57:26 -0400 Subject: [PATCH 05/15] feat: push settings between panels (2c), and fix edges 500ing on every dataset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two things, found together: 2c is the last functional stage of docs/split_screen_phase2.md, and verifying it surfaced a regression I shipped in ab9d739 that had broken every /edges endpoint. ## The regression `DUCKDB_MEMORY_LIMIT` lost its fixed 8GB default in v0.8.4 so the cap could be sized from memory actually available, and both compose files now pass it through empty. But `edge_reader.py` kept its own connection setup reading `os.getenv("DUCKDB_MEMORY_LIMIT", "8GB")` — and that default applies only when the variable is *absent*. Set-but-empty yields "", so DuckDB got `SET memory_limit=''` and raised ParserException: Parser Error: Memory limit must have a number on every edge query, in the dev server and the container alike. Edges, the tissue graph and the LRM catalogue were dead on all nine datasets. It did not show up earlier because the failure needs the variable *set to empty*, which only the compose files do — a bare `python3 tests/golden_snapshot.py` inherits an unset variable and passes. The real defect is two modules defaulting one environment variable two different ways, so EdgeReader now goes through `duck.connect()`. It also picks up the temp_directory it never had, so a large edge query can spill rather than fail. `duck.py` additionally strips the value, since a whitespace-only setting is truthy and would reach DuckDB unchanged — a hole the new check found. `backend/tests/duckdb_config_check.py` covers unset / empty / blank / explicit against a real edge query. The golden snapshot structurally cannot catch this. ## 2c `pushSettings(from, to, allowed)` copies one panel's settings onto the other, sanitised for the target. The button sits under the link toggle, unlinked only, labelled with its direction. The guard is the substance: a filter naming a column the target lacks 400s on every viewport change and the panel silently stops rendering, so those settings are dropped rather than copied. Column names come from /cells/schema and /edges/schema, which the store never fetches — the component gathers the vocabulary so the store action stays pure and testable. A gene allowlist with no overlap becomes null ("no filter") rather than an empty Set ("show no species"), matching a dataset change; colour clamps reset across different datasets, since [0, 4000] onto data topping out at 70 paints everything one colour. Verified across MERSCOPE → CosMx (zero gene overlap) through the real button: geometry and palette copied, gene allowlist and clamp dropped, source untouched, target still rendering 5000/38996 cells. 46 frontend tests, golden guard 238/238. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 7 +- backend/app/readers/duck.py | 5 +- backend/app/readers/edge_reader.py | 23 +++-- backend/tests/duckdb_config_check.py | 96 ++++++++++++++++++ docs/split_screen_phase2.md | 24 +++-- frontend/src/components/LayerPanel.jsx | 73 ++++++++++++++ frontend/src/store.js | 79 +++++++++++++++ frontend/src/store.settings.test.js | 133 +++++++++++++++++++++++++ 8 files changed, 424 insertions(+), 16 deletions(-) create mode 100644 backend/tests/duckdb_config_check.py diff --git a/CLAUDE.md b/CLAUDE.md index ab55668..ae5386e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -572,8 +572,11 @@ Style and choice settings — layer opacity, palettes, colour-by, filters, LRM selection, edge geometry — are **shared by default** and can be unlinked per panel from the sidebar tabs (Phase 2a/2b). One sidebar driving both panels is what makes a side-by-side comparison comparable, so linked stays the default. -See `docs/split_screen_phase2.md`; the remaining stage is 2c, an explicit -push-settings-to-the-other-panel button. The sidebar +A `copy panel N → panel M` button under the toggle pushes one panel's settings +onto the other in one shot, dropping any that name a column, gene or mechanism +the target dataset lacks — an inherited filter on a missing column 400s on every +viewport change and the panel silently stops rendering. See +`docs/split_screen_phase2.md`; only the docs pass (2e) remains. The sidebar reconciles across panels with `hooks/usePanels.js`, whose rule is **union, then degrade per panel**: offer a control if *either* panel can use it, and let the panel that cannot render nothing. Intersecting instead would hide controls that diff --git a/backend/app/readers/duck.py b/backend/app/readers/duck.py index 7d7f2f6..76c4008 100644 --- a/backend/app/readers/duck.py +++ b/backend/app/readers/duck.py @@ -67,7 +67,10 @@ def _default_memory_limit() -> str: return f"{mb}MB" -_MEMORY_LIMIT = os.getenv("DUCKDB_MEMORY_LIMIT") or _default_memory_limit() +# .strip() matters: an unset variable and one set to "" or " " must all fall +# through to the computed default. A whitespace value is truthy, so without it +# DuckDB receives `SET memory_limit=' '` and raises a ParserException. +_MEMORY_LIMIT = (os.getenv("DUCKDB_MEMORY_LIMIT") or "").strip() or _default_memory_limit() _THREADS = os.getenv("DUCKDB_THREADS", "4") diff --git a/backend/app/readers/edge_reader.py b/backend/app/readers/edge_reader.py index 6896852..57a29b0 100644 --- a/backend/app/readers/edge_reader.py +++ b/backend/app/readers/edge_reader.py @@ -22,7 +22,6 @@ from app.readers import duck, metadata_filter, supplemental from app.readers.metadata_filter import MetadataFilter -_DUCKDB_MEMORY_LIMIT = os.getenv("DUCKDB_MEMORY_LIMIT", "8GB") _UNSET = object() @@ -86,12 +85,22 @@ def _from(self) -> str: return f"read_parquet('{self._sql_path}')" def _conn(self): - # Each call gets a fresh, isolated connection — duckdb's default connection - # is not thread-safe and causes empty/corrupt results under FastAPI concurrency. - conn = duckdb.connect() - conn.execute(f"SET memory_limit='{_DUCKDB_MEMORY_LIMIT}'") - conn.execute("SET threads=4") - return conn + # Shared with the spatial readers via duck.connect(), which also gives a + # fresh isolated connection per call (duckdb's default connection is not + # thread-safe and returns empty or corrupt results under FastAPI's + # threadpool rather than raising). + # + # This used to build its own connection with + # `os.getenv("DUCKDB_MEMORY_LIMIT", "8GB")`. That default only applies + # when the variable is *absent*: set-but-empty returns "", so + # `SET memory_limit=''` raised a ParserException and every /edges + # endpoint 500'd. Compose sets it empty on purpose now, to let the limit + # be sized from available memory — so the two disagreed and edges stopped + # loading everywhere. Going through duck.connect() means there is one + # defaulting rule instead of two, and EdgeReader also picks up the + # temp_directory it never had, so a large edge query can spill instead of + # failing outright. + return duck.connect() def schema(self) -> dict: """Column names and dtypes, parquet columns plus any supplemental ones. diff --git a/backend/tests/duckdb_config_check.py b/backend/tests/duckdb_config_check.py new file mode 100644 index 0000000..7b6a94c --- /dev/null +++ b/backend/tests/duckdb_config_check.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +""" +DuckDB is configured correctly however DUCKDB_MEMORY_LIMIT is (or is not) set. + +Why this exists +--------------- +`DUCKDB_MEMORY_LIMIT` stopped having a fixed default in v0.8.4, because a flat +`8GB` on a stock ~8 GB Docker Desktop VM let DuckDB take all of RAM and the +spatial-index build was OOM-killed mid-request. Both compose files now pass it +through empty so the limit is sized from memory actually available. + +That broke every `/edges` endpoint. `edge_reader.py` had its own connection +setup reading `os.getenv("DUCKDB_MEMORY_LIMIT", "8GB")` — and that default only +applies when the variable is *absent*. Set-but-empty yields `""`, so the driver +got `SET memory_limit=''` and raised + + duckdb.duckdb.ParserException: Parser Error: Memory limit must have a number + +on every edge query. Two modules defaulting the same environment variable two +different ways is the actual defect; EdgeReader now goes through +`duck.connect()`, so there is one rule. + +The golden snapshot cannot catch this: it exercises readers in whatever +environment it happens to run in, and the failure only appears when the variable +is set to an empty string. + + python3 tests/duckdb_config_check.py +""" +import os +import sys +import traceback +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +DATA_ROOT = Path(os.getenv("DATA_ROOT", Path(__file__).resolve().parents[2] / "sample_data")) + +# The values that have actually appeared in a compose file or a shell, plus the +# shapes a user might reasonably put in .env.prod. +CASES = [ + ("unset", None), + ("empty", ""), # ${DUCKDB_MEMORY_LIMIT:-} — the one that broke edges + ("blank", " "), + ("explicit", "512MB"), +] + +failures = [] + + +def check_connect(label): + """A connection must be usable, whatever the environment says.""" + import importlib + from app.readers import duck + importlib.reload(duck) + limit = duck._MEMORY_LIMIT + with duck.connect() as conn: + conn.execute("SELECT 1").fetchone() + # A bare number is not a valid DuckDB memory limit either. + if not any(limit.upper().endswith(u) for u in ("KB", "MB", "GB", "TB", "B")): + failures.append(f"[{label}] memory_limit {limit!r} has no unit suffix") + return limit + + +def check_edges(label): + """Every edge endpoint shares one connection helper — exercise a real query.""" + from app.readers.edge_reader import EdgeReader + found = sorted(DATA_ROOT.glob("*/edges.parquet")) + if not found: + return "no edge datasets present — skipped" + src = found[0] + reader = EdgeReader(src, pixel_size=1.0) + n = len(reader.lrm_catalogue()) + reader.query_grouped(bbox=None, density=1.0) + return f"{src.parent.name}: {n} LRMs" + + +for label, value in CASES: + if value is None: + os.environ.pop("DUCKDB_MEMORY_LIMIT", None) + else: + os.environ["DUCKDB_MEMORY_LIMIT"] = value + try: + limit = check_connect(label) + detail = check_edges(label) + print(f" OK {label:<9} {str(value)!r:<10} -> memory_limit={limit:<10} {detail}") + except Exception as exc: + failures.append(f"[{label}] {type(exc).__name__}: {exc}") + print(f" FAIL {label:<9} {str(value)!r:<10} -> {type(exc).__name__}: {exc}") + traceback.print_exc() + +print() +if failures: + for f in failures: + print(" ✗", f) + sys.exit(1) +print("DuckDB configuration is valid for every DUCKDB_MEMORY_LIMIT form.") diff --git a/docs/split_screen_phase2.md b/docs/split_screen_phase2.md index c51d17c..a25abc6 100644 --- a/docs/split_screen_phase2.md +++ b/docs/split_screen_phase2.md @@ -171,12 +171,24 @@ Two decisions worth knowing: Identical while linked; unlinked, the active panel's gene selection would describe the wrong cell. -**2c — push settings.** `pushSettings(from, to)` deep-copies -`panels[from].settings` into `panels[to]`. One button per panel header, labelled -with its direction. Needs a guard: settings naming a column, gene or LRM that the -target dataset does not have must be dropped rather than copied, or the target -panel starts 400ing on every viewport change — the same failure mode as -problem 1. Validate against the target's schema/gene list before writing. +**2c — push settings. DONE.** `pushSettings(from, to, allowed)` deep-copies the +source panel's settings onto the target, sanitised by `sanitiseSettings`. The +button lives under the link toggle and appears only when the panels are unlinked, +since pushing between linked panels is a no-op; its label names the direction +(`copy panel 1 → panel 2`). + +The guard is the substance. Settings naming a column, gene or mechanism the +target lacks are dropped, because a `cellFilter` on a missing column makes the +backend 400 on *every* viewport change — the panel stops rendering with no +indication why. Column names come from `/cells/schema` and `/edges/schema`, which +the store never fetches, so the component gathers the vocabulary and the store +action stays pure and testable; genes and LRMs it already holds per panel. + +Two rules worth knowing: a gene allowlist with no overlap falls back to `null` +("no filter") rather than an empty Set ("show no species"), matching what a +dataset change leaves behind; and the colour clamps reset when the datasets +differ, since a range in one dataset's units — [0, 4000] onto data topping out at +70 — paints everything the bottom colour and reads as a broken render. **2d — narrow the dataset-change reset. DONE, folded into 2b.** Sequencing it last turned out to be wrong. 2b makes the promise "editing panel 1 only", and a diff --git a/frontend/src/components/LayerPanel.jsx b/frontend/src/components/LayerPanel.jsx index 46f8a4b..ca384f8 100644 --- a/frontend/src/components/LayerPanel.jsx +++ b/frontend/src/components/LayerPanel.jsx @@ -850,10 +850,83 @@ function PanelTabs() { : `editing panel ${activePanel + 1} only`} + {/* Only meaningful once the panels can differ. */} + {!linkSettings && } ); } +/** + * One-shot copy of the active panel's settings onto the other. + * + * The explicit half of the original request: explore either side, then force + * the other to match. Unlike re-linking, the panels stay independent after, so + * you can push a baseline across and then diverge again from it. + * + * Settings naming a column, gene or mechanism the target does not have are + * dropped before writing — see sanitiseSettings. That check is here rather than + * in the store because the column names come from /cells/schema and + * /edges/schema, which the store never fetches. + */ +function PushSettingsButton() { + const apiBase = useStore((s) => s.apiBase); + const activePanel = useStore((s) => s.activePanel); + const panels = useStore((s) => s.panels); + const pushSettings = useStore((s) => s.pushSettings); + const [busy, setBusy] = useState(false); + const [done, setDone] = useState(false); + + const from = activePanel; + const to = activePanel === 0 ? 1 : 0; + const target = panels[to]; + + const run = async () => { + if (!target?.dataset) return; + setBusy(true); + const cols = async (url) => { + try { + const r = await fetch(url); + if (!r.ok) return null; + return new Set(Object.keys((await r.json())?.columns ?? {})); + } catch { return null; } + }; + const ef = `?edge_file=${encodeURIComponent(target.edgeFile)}`; + const [cellFields, edgeFields] = await Promise.all([ + cols(`${apiBase}/spatial/${target.dataset}/cells/schema`), + cols(`${apiBase}/edges/${target.dataset}/schema${ef}`), + ]); + pushSettings(from, to, { + cellFields, + edgeFields, + // Already in the store, per panel — no fetch needed. + genes: target.allGenes?.length ? new Set(target.allGenes) : null, + lrms: target.lrmCatalogue?.length + ? new Set(target.lrmCatalogue.map((e) => e.lrm ?? `${e.ligand}|${e.receptor}`)) + : null, + }); + setBusy(false); + setDone(true); + setTimeout(() => setDone(false), 1500); + }; + + return ( + + ); +} + function LinkColorScaleRow() { const { linkColorScale, setLinkColorScale } = usePanelSettings(); return ( diff --git a/frontend/src/store.js b/frontend/src/store.js index cb6eef6..1f80058 100644 --- a/frontend/src/store.js +++ b/frontend/src/store.js @@ -82,6 +82,67 @@ function cloneSettings(src) { }; } +/** + * Strip settings that name something the target dataset does not have. + * + * Copying settings across datasets is where this gets dangerous rather than + * merely wrong-looking: a `cellFilter` naming a column the target lacks makes + * the backend return 400 on *every* viewport change, so the panel stops + * rendering entirely and the cause is invisible from the UI. Dropping the + * setting degrades to "no filter", which is recoverable and obvious. + * + * `allowed` supplies the target's vocabulary — `{cellFields, edgeFields, genes, + * lrms}`, each a Set or null to skip that check. The caller provides it because + * metadata columns come from /cells/schema and /edges/schema, which the store + * does not fetch; genes and LRMs it already holds per panel. + * + * `sameDataset` governs the colour clamps. A clamp is a range in the *source* + * data's units — carrying [0, 4000] onto a dataset topping out at 70 paints + * everything the bottom colour, which reads as a broken render rather than a + * copied setting. Same dataset, the clamp is exactly what you meant to copy. + */ +export function sanitiseSettings(next, allowed, sameDataset) { + const { cellFields = null, edgeFields = null, genes = null, lrms = null } = allowed ?? {}; + const has = (set, v) => set === null || set.has(v); + const filterKeys = (obj, keep) => + Object.fromEntries(Object.entries(obj).filter(([k]) => keep(k))); + + if (next.colorBy?.mode === "metadata" && !has(cellFields, next.colorBy.field)) { + next.colorBy = { mode: "off", field: null }; + } + if (next.edgeColorBy?.mode === "metadata" && !has(edgeFields, next.edgeColorBy.field)) { + next.edgeColorBy = { mode: "lrm_set", field: null }; + } + if (next.cellFilter && !has(cellFields, next.cellFilter.field)) next.cellFilter = null; + if (next.edgeFilter && !has(edgeFields, next.edgeFilter.field)) next.edgeFilter = null; + + if (next.selectedGenes && genes) { + const keep = [...next.selectedGenes].filter((g) => genes.has(g)); + // No overlap at all means the allowlist says nothing about this panel. + // Falling back to "no filter" matches what a dataset change does; an empty + // Set would mean "show no species", which is a stranger thing to inherit. + next.selectedGenes = keep.length ? new Set(keep) : null; + } + if (next.hiddenLrms && lrms) { + next.hiddenLrms = new Set([...next.hiddenLrms].filter((l) => lrms.has(l))); + } + + next.categoricalOverrides = filterKeys(next.categoricalOverrides, (k) => { + const [scope, field] = k.split("::"); + return has(scope === "edge" ? edgeFields : cellFields, field); + }); + next.categoryColorOverrides = filterKeys(next.categoryColorOverrides, + (k) => has(cellFields, k.split("::")[0])); + next.transcriptColorOverrides = filterKeys(next.transcriptColorOverrides, + (k) => has(genes, k)); + + if (!sameDataset) { + next.cellColorClamp = { low: null, high: null }; + next.edgeColorClamp = { low: null, high: null }; + } + return next; +} + /** * One panel's dataset-bound state. * @@ -316,6 +377,24 @@ export const useStore = create((set, get) => ({ }; }), + /** + * Copy one panel's settings onto another. + * + * The explicit form of what the link toggle does continuously: explore either + * side freely, then force the other to match. Unlike linking it is one-shot, + * so the panels stay independent afterwards. + * + * `allowed` is the target's vocabulary; see sanitiseSettings. Passing null + * copies verbatim, which is only safe when both panels show the same dataset. + */ + pushSettings: (from, to, allowed = null) => set((s) => { + const src = s.panels[from]?.settings; + if (!src || !s.panels[to] || from === to) return {}; + const sameDataset = s.panels[from].dataset === s.panels[to].dataset; + const next = sanitiseSettings(cloneSettings(src), allowed, sameDataset); + return { panels: s.panels.map((p, i) => (i === to ? { ...p, settings: next } : p)) }; + }), + /** * Merge a patch into panel settings. * diff --git a/frontend/src/store.settings.test.js b/frontend/src/store.settings.test.js index 4057c8c..0b8c9a0 100644 --- a/frontend/src/store.settings.test.js +++ b/frontend/src/store.settings.test.js @@ -271,3 +271,136 @@ describe("2b — re-linking adopts the active panel's settings", () => { expect(settings(1).edgeWidth).toBe(before); }); }); + +describe("2c — pushSettings copies one panel onto the other", () => { + beforeEach(() => { + S().setLinkSettings(false); + S().setActivePanel(0); + S().setEdgeWidth(8); + S().setCellColorPalette("magma"); + S().setLayerProp("edges", "visible", true); + }); + + it("copies dataset-independent settings verbatim", () => { + S().pushSettings(0, 1); + expect(settings(1).edgeWidth).toBe(8); + expect(settings(1).cellColorPalette).toBe("magma"); + expect(settings(1).layers.edges.visible).toBe(true); + }); + + it("leaves the source untouched and the panels independent", () => { + S().pushSettings(0, 1); + S().setActivePanel(1); + S().setEdgeWidth(1); + expect(settings(0).edgeWidth).toBe(8); + // A shallow copy would alias the containers and write through. + S().setLayerProp("edges", "visible", false); + expect(settings(0).layers.edges.visible).toBe(true); + }); + + it("is a no-op onto itself", () => { + S().pushSettings(0, 0); + expect(settings(0).edgeWidth).toBe(8); + }); +}); + +describe("2c — the guard drops what the target cannot honour", () => { + const allowed = { + cellFields: new Set(["cluster"]), + edgeFields: new Set(["confidence"]), + genes: new Set(["A", "B"]), + lrms: new Set(["L1|R1"]), + }; + + beforeEach(() => { + S().setLinkSettings(false); + S().setActivePanel(0); + // Give panel 1 a different dataset so the clamp rule engages too. + useStore.setState({ + panels: S().panels.map((p, i) => ({ ...p, dataset: i === 0 ? "src" : "dst" })), + }); + }); + + it("drops a cell filter naming a missing column", () => { + // The dangerous one: the backend 400s on every viewport change, so the + // panel renders nothing and the UI gives no clue why. + S().setCellFilter({ field: "absent_column", values: ["x"] }); + S().pushSettings(0, 1, allowed); + expect(settings(1).cellFilter).toBeNull(); + }); + + it("keeps a cell filter the target does have", () => { + S().setCellFilter({ field: "cluster", values: ["4"] }); + S().pushSettings(0, 1, allowed); + expect(settings(1).cellFilter).toEqual({ field: "cluster", values: ["4"] }); + }); + + it("turns colour-by off when it names a missing column", () => { + S().setColorBy("metadata", "absent_column"); + S().pushSettings(0, 1, allowed); + expect(settings(1).colorBy).toEqual({ mode: "off", field: null }); + }); + + it("leaves a non-metadata colour-by alone", () => { + S().setColorBy("gene_set", null); + S().pushSettings(0, 1, allowed); + expect(settings(1).colorBy).toEqual({ mode: "gene_set", field: null }); + }); + + it("drops an edge filter and edge colour-by naming missing columns", () => { + S().setEdgeFilter({ field: "nope", values: ["1"] }); + S().setEdgeColorBy("metadata", "nope"); + S().pushSettings(0, 1, allowed); + expect(settings(1).edgeFilter).toBeNull(); + expect(settings(1).edgeColorBy).toEqual({ mode: "lrm_set", field: null }); + }); + + it("intersects the gene allowlist", () => { + S().setSelectedGenes(new Set(["A", "ZZZ"])); + S().pushSettings(0, 1, allowed); + expect([...settings(1).selectedGenes]).toEqual(["A"]); + }); + + it("falls back to no filter when no selected gene exists in the target", () => { + // An empty Set would mean "show no species", a stranger thing to inherit + // than "no filter" — which is also what a dataset change leaves behind. + S().setSelectedGenes(new Set(["ZZZ"])); + S().pushSettings(0, 1, allowed); + expect(settings(1).selectedGenes).toBeNull(); + }); + + it("intersects hidden mechanisms", () => { + S().toggleLrm("L1|R1"); + S().toggleLrm("L9|R9"); + S().pushSettings(0, 1, allowed); + expect([...settings(1).hiddenLrms]).toEqual(["L1|R1"]); + }); + + it("drops keyed overrides whose column or gene is absent", () => { + S().setCategoricalOverride("cell", "cluster", true); + S().setCategoricalOverride("cell", "absent_column", true); + S().setCategoryColorOverride("cluster", "4", [1, 2, 3, 255]); + S().setCategoryColorOverride("absent_column", "x", [1, 2, 3, 255]); + S().setTranscriptColorOverride("A", [1, 2, 3, 255]); + S().setTranscriptColorOverride("ZZZ", [1, 2, 3, 255]); + S().pushSettings(0, 1, allowed); + expect(Object.keys(settings(1).categoricalOverrides)).toEqual(["cell::cluster"]); + expect(Object.keys(settings(1).categoryColorOverrides)).toEqual(["cluster::4"]); + expect(Object.keys(settings(1).transcriptColorOverrides)).toEqual(["A"]); + }); + + it("resets colour clamps across different datasets, keeps them within one", () => { + // A clamp is a range in the source data's units. [0,4000] onto a dataset + // topping out at 70 paints everything the bottom colour, which reads as a + // broken render rather than a copied setting. + S().setCellColorClamp(0, 4000); + S().pushSettings(0, 1, allowed); + expect(settings(1).cellColorClamp).toEqual({ low: null, high: null }); + + useStore.setState({ panels: S().panels.map((p) => ({ ...p, dataset: "same" })) }); + S().setActivePanel(0); + S().setCellColorClamp(0, 4000); + S().pushSettings(0, 1, allowed); + expect(settings(1).cellColorClamp).toEqual({ low: 0, high: 4000 }); + }); +}); From 0d55f0363d54c41f453df346e2cdbbe7b80215d8 Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Sun, 16 Aug 2026 09:03:39 -0400 Subject: [PATCH 06/15] docs: manual describes split screen as it now works (2e) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Last stage of docs/split_screen_phase2.md, which is now closed out. The hosted manual needed correcting rather than extending. It said "Both panels share all layer settings (visibility, color-by, LRM filter, etc.) but have independent pan and zoom positions" — untrue since Phase 1 gave each panel its own dataset, and wrong through two releases. Someone reading it would not have known the feature they were looking for existed. The Split-Screen section now covers two datasets side by side, the panel tabs and link checkbox, the copy button and what it declines to copy, micron-based zoom matching, and the shared colour scale. Two points get called out as notes because they are surprising rather than discoverable: re-linking re-syncs to the tab you are on, so pick that tab first; and Match works in microns, which is what makes 20% of a 6.5 mm Visium capture area comparable to 20% of a 55 µm seqFISH region rather than fifty-fold apart. Also corrected nearby claims that the same work invalidated: Clear removes that panel's annotations rather than everything, annotations belong to the panel that drew them, and the dataset/image/edge pickers move into the panel headers in split mode. The dataset-picker section gained a note on what a dataset change clears and what it keeps, since that surprised me while writing the tests. Verified rendered in a browser rather than by reading the source: all five headings present, the old claim gone, anchors resolve, HTML balanced. Co-Authored-By: Claude Opus 5 --- docs/index.html | 93 +++++++++++++++++++++++++++++++++---- docs/split_screen_phase2.md | 14 ++++-- 2 files changed, 95 insertions(+), 12 deletions(-) diff --git a/docs/index.html b/docs/index.html index 2a502c6..c4d72e4 100644 --- a/docs/index.html +++ b/docs/index.html @@ -758,6 +758,19 @@

Dataset & Image Picker

image is viewed, TissuePlex builds a tile pyramid (10–30 seconds); if the canvas looks blank, wait and refresh.

+

In split-screen mode these dropdowns move into + each panel's own header instead, because the two panels can show different + datasets and an image or edge file only means something relative to one of + them.

+ +
+

Changing a panel's dataset clears the settings that name a column, gene or + mechanism — metadata filters, the color-by field, the gene selection, the + mechanism checklist. It has to: those names come from the dataset you just + left. Sliders, palettes and layer visibility are kept. While the panels are + linked, this clears both; unlinked, only the panel you changed.

+
+
Tip: TissuePlex works best in Google Chrome or Firefox. Switch to Chrome @@ -1129,22 +1142,75 @@

Clicking on an Edge or Tissue Graph Line

Split-Screen Mode

-

Split-screen mode divides the viewer into two independent side-by-side - panels, each with its own pan and zoom position. Use it to compare two tissue - regions or to view the same region with different filter settings.

+

Split-screen mode divides the viewer into two side-by-side panels. Each has + its own pan and zoom, and each can show a different dataset — + so you can put two samples, two platforms, or two timepoints next to each + other. Both panels can also show the same dataset rendered two different ways.

  • Click ⊞ Split in the annotation toolbar to enter split-screen mode.
  • Click □ Single to return to a single panel.
    • -
  • Both panels share all layer settings (visibility, color-by, LRM filter, - etc.) but have independent pan and zoom positions.
    • +
  • In split mode each panel gets its own dataset, image and edge-source + picker in its header, since those only mean something relative to one + dataset.
+

Choosing Which Panel You Are Editing

+

The sidebar drives both panels. At the top of it, + Panel 1 / Panel 2 tabs choose + which one your edits apply to, and a checkbox below them controls whether they + are linked:

+ +
    +
  • Linked (the default): every change — opacity, palette, + color-by, filters, mechanism selection, edge geometry — applies to both + panels at once. This is what makes a side-by-side comparison actually + comparable.
    • +
  • Unlinked: edits apply only to the tab you are on. Use + this to show cluster coloring on one side and gene expression on the other, + or two palettes, or two filters.
    • +
+ +
+

Switching the checkbox back on re-syncs both panels to the + tab you are currently on. That is deliberate: a control labelled "linked" + over two visibly different panels would not be telling you the truth. Pick + the tab whose look you want to keep before you re-link.

+
+ +

Copying Settings From One Panel to the Other

+

While the panels are unlinked, a + copy panel 1 → panel 2 button appears under the + checkbox. It copies every display setting from the tab you are on to the other + panel in one go, and then leaves the panels independent again — so you can + establish a common baseline and diverge from it.

+ +

Settings that name something the other dataset does not have are dropped + rather than copied: a metadata filter on a column it lacks, a gene allowlist + with no genes in common, mechanisms absent from its edge file. Color range + limits also reset when the two panels show different datasets, because a range + measured in one dataset's units is meaningless in another's.

+

Matching Zoom Between Panels

The ⇔ Match button sets the other panel to your - current panel's zoom level while each panel remains centered on its own - region.

+ current panel's zoom while each panel stays centered on its own region.

+ +
+

Match works in microns, not pixels, so the same physical + distance spans the same width on screen in both panels — exactly as a scale + bar would. This matters when the panels hold different platforms: 20% of a + 6.5 mm Visium capture area and 20% of a 55 µm seqFISH region differ + by more than fifty-fold.

+
+ +

Color Scales Across Panels

+

The shared colour scale across panels checkbox, on + by default, makes both panels map color through one range computed across both. + Leave it on for any figure. Two panels that each auto-scaled to their own data + look comparable and are not — one panel's yellow might be 40 counts and + the other's 4,000. Turn it off only when one panel's range is so much narrower + that sharing flattens it, and say so in the legend.

@@ -1162,8 +1228,17 @@

Annotation Tools

distance between them in micrometers. -

The Clear button removes all regions and - measurements at once.

+

The Clear button removes the regions and + measurements belonging to that panel — each panel has its own toolbar, + so clearing one leaves the other's work alone.

+ +
+

In split-screen mode, regions and measurements belong to the panel they + were drawn in and appear only there. They are stored in that dataset's + coordinates, so a region drawn on one tissue would land somewhere meaningless + if it were drawn on the other. Exported cell lists and measured distances + likewise resolve against the panel that produced them.

+

Exporting Cells Within a Region

Each drawn region appears in the Layer Panel's Regions diff --git a/docs/split_screen_phase2.md b/docs/split_screen_phase2.md index a25abc6..05a511d 100644 --- a/docs/split_screen_phase2.md +++ b/docs/split_screen_phase2.md @@ -1,6 +1,7 @@ # Split screen, Phase 2 — per-panel settings -Status: **planned, not started.** Phase 1 shipped in v0.8.4 (PR #56). +Status: **complete.** Phase 1 shipped in v0.8.4 (PR #56); stages 2a–2e followed. +Kept as the record of what was decided and why, not as outstanding work. Phase 1 made the two panels able to show two *different datasets*. Phase 2 makes their *settings* independent, with an explicit way to re-link them. This document @@ -197,8 +198,15 @@ global reset breaks it: an action on panel 2 still destroys panel 1's work, so — changed is always reset; the others only while linked, where they share one set of values and a stale filter would 400 on every viewport change. -**2e — docs.** CLAUDE.md's Split-Screen section, and `docs/index.html` (the -hosted manual) for the tabs, the link toggle and the push button. +**2e — docs. DONE.** CLAUDE.md was updated as each stage landed. +`docs/index.html`, the hosted manual, needed more than an addition: it claimed +"Both panels share all layer settings … but have independent pan and zoom +positions", which stopped being true at Phase 1 and had been wrong through two +releases. It now covers two datasets side by side, the panel tabs and link +toggle (including that re-linking re-syncs to the active tab), the copy button +and what it drops, micron-based zoom matching, the shared colour scale, and +per-panel annotations. The dataset-picker section gained a note that the pickers +move into the panel headers in split mode and what a dataset change clears. ## Risks From 23c2207d13ae9a329619e472d599adf64520d5c4 Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Sun, 16 Aug 2026 09:56:48 -0400 Subject: [PATCH 07/15] chore: v0.8.5 Co-Authored-By: Claude Opus 5 --- backend/app/main.py | 2 +- frontend/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app/main.py b/backend/app/main.py index ee1c810..26a8700 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -3,7 +3,7 @@ from app.routers import tiles, spatial, edges, layers -APP_VERSION = "0.8.4" +APP_VERSION = "0.8.5" app = FastAPI(title="TissuePlex API", version=APP_VERSION) diff --git a/frontend/package.json b/frontend/package.json index abb0d1b..99c0f97 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "tissueplex", - "version": "0.8.4", + "version": "0.8.5", "private": true, "scripts": { "dev": "vite", From a9627bc0e15d07b3b5d4fce9e85c36c562c87b66 Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Tue, 18 Aug 2026 08:12:53 -0400 Subject: [PATCH 08/15] =?UTF-8?q?docs:=20plan=20for=20issue=20#59=20?= =?UTF-8?q?=E2=80=94=20independent=20sending/receiving=20edge=20filters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Writes up #59 before touching code, as with split-screen Phase 2. Three lab directives shape it beyond the issue text, and two of them change current behaviour: - The tissue graph is ground truth: shown or hidden, never subset by a filter. Today one useEdges request feeds both the graph and the edge layer, so filtering edges thins the structural graph too. - Cell and edge filtration are completely independent. Today query_grouped(cell_ids=…) emits `sending IN S AND receiving IN S`, so filtering cells removes edges. - Density is applied last, being a rendering control rather than a selection. This one already holds — the grouped query samples an outer select wrapping the filtered, grouped subquery — and the plan records it as an invariant to preserve rather than work to do. Findings that shaped the design, verified rather than assumed: - Half of #59 already ships. sending_type/receiving_type are populated columns and are already offered in the edge filter dropdown; on mouse_ileum_tiny, sending_type=Fibroblast gives 55 of 223 edges with one sending type and all five receiving types. The blocker is that edgeFilter holds exactly one filter — the composition gap deferred in #45 — not the sending/receiving distinction. - The tissue graph needs its own fetch rather than a passes_filter flag on the shared result, and density-last is why: the two layers want opposite orderings of the same pipeline, and a flag would be read after sampling had already thinned the rows it describes. - Efficiency, which the issue asks about directly: one hash semi-join becomes two, on a query that already performs one. filter_cell_ids() and duck.register_ids() already exist. Sizing measured across the bundled datasets — 169,219 unique edges on cosmx-mousebrain is the largest. Rendering that is not the constraint; the ~15 fields per edge in the payload are, hence a lean projection staged as optional work to be measured first. CLAUDE.md indexes the plan and flags the both-endpoints rule as slated for reversal, keeping the original reasoning since it still explains the code. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 15 +- docs/edge_filter_independence.md | 230 +++++++++++++++++++++++++++++++ 2 files changed, 243 insertions(+), 2 deletions(-) create mode 100644 docs/edge_filter_independence.md diff --git a/CLAUDE.md b/CLAUDE.md index ae5386e..accce1b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -204,8 +204,12 @@ docs/ public_datasets.md Public datasets used for development, and the classic Visium pair the reader was verified against split_screen_phase2.md Spec for making panel *settings* per-panel. Phase 1 - (per-panel datasets) shipped in v0.8.4; this is what - remains. Read before touching store.js's shared state. + (per-panel datasets) shipped in v0.8.4; stages 2a–2e + shipped in v0.8.5. Kept as the design record. + edge_filter_independence.md Plan for issue #59. Decouples the tissue graph and + the cell filter from edge filtering, then adds + independent sending/receiving filters. Not started — + read before touching the edge filter path. index.html The user manual, published to GitHub Pages at https://raredonlab.github.io/TissuePlex/ — hand-written HTML, no build step. Update it when a UI control changes. @@ -1222,6 +1226,13 @@ subset renders at full density. column is in the parquet, and a semi-join against a registered frame when it comes from `edge-metadata/`. + **The both-endpoints rule is slated to be reversed.** The lab's position is that + cell and edge filtration should be completely independent, and that the tissue + graph is ground truth that filtering must never subset — today one `useEdges` + request feeds both the graph and the edge layer, so both couplings are live. See + `docs/edge_filter_independence.md` (issue #59). The reasoning above is the decision + being reversed, kept because it explains what the code currently does. + **Large id sets go through `duck.register_ids()`, not `IN (?, ?, …)`.** A filter can keep hundreds of thousands of cells; binding that many parameters is unworkable and the SQL text alone reaches megabytes. Registering a one-column frame makes it an diff --git a/docs/edge_filter_independence.md b/docs/edge_filter_independence.md new file mode 100644 index 0000000..ebbaa54 --- /dev/null +++ b/docs/edge_filter_independence.md @@ -0,0 +1,230 @@ +# Issue #59 — independent sending / receiving edge filters + +Status: **planned, not started.** + +Upstream ask +([#59](https://github.com/RaredonLab/TissuePlex/issues/59)): designate a sending +cell type and a receiving cell type independently, and see the resulting graph +structure. The issue asks whether deriving this from *cell* metadata per edge is +efficient. It is — the machinery already exists and the added cost is one extra +hash semi-join. + +Two directives from the lab shape this beyond what the issue text says, and both +change current behaviour: + +1. **The tissue graph is ground truth and must not be filtered.** It is the total + set of edges, shown or hidden, never subset. Only the edge-data layer drawn on + top of it responds to filters. +2. **Cell filtration and edge filtration are completely independent.** Filtering + cells must not remove edges, and filtering edges must not remove cells. +3. **Density is applied last, after every other filter.** It is a rendering-volume + control, not a selection criterion. + +--- + +## What already works + +Half of #59 ships today and is worth knowing before building anything. +`sending_type` and `receiving_type` are real populated columns in +`edges.parquet`, they are not in `EDGE_FILTER_SKIP`, so the edge filter dropdown +already offers them. Measured on `mouse_ileum_tiny`: + +| filter | edges | sending types in result | receiving types in result | +|---|---|---|---| +| none | 223 | all five | all five | +| `sending_type = Fibroblast` | 55 | `Fibroblast` only | all five | + +So one-sided filtering works. **The blocker is that `edgeFilter` holds exactly +one filter** — the composition gap deliberately deferred in issue #45 — so +"sending = Fibroblast *and* receiving = Endothelial" cannot be expressed. That +limit, not the sending/receiving distinction, is what needs fixing. + +## Directive 3 already holds — preserve it + +Density is already last on the edge path, and this is worth stating as an +invariant because it is easy to break by accident. The grouped query samples the +*outer* select, wrapping the filtered and grouped subquery: + +```sql +SELECT * FROM ( + SELECT … + FROM edges + WHERE + GROUP BY edge + HAVING lrm_count >= 1 +) USING SAMPLE PERCENT (bernoulli) +LIMIT 500000 +``` + +Everything selective runs first; the sample then draws from whatever survived, so +narrowing to a rare subset renders that subset at full density instead of a tenth +of it. The nesting is load-bearing — applied alongside the `WHERE`, DuckDB is free +to sample before filtering, which silently reintroduces the problem. + +Every new predicate in this plan therefore belongs in the same `WHERE` clause, not +in post-processing, and the new tissue-graph endpoint must keep the same shape. + +**This is also why the tissue graph needs its own query rather than a flag on the +shared one** — see below. Filter-then-sample and don't-filter-then-sample are +different orderings of the same pipeline, and one statement cannot do both. + +## The three current couplings to break + +`useEdges` issues **one** request whose result feeds **both** the tissue-graph +layer and the directed-edge layer (`Viewer.jsx`: `id: "tissue-graph"`, +`data: allDirectedEdges`). That request carries `cell_filter`, `edge_filter` and +`density`. Consequently, today: + +- filtering **cells** removes edges, because `query_grouped(cell_ids=…)` emits + `sending_cell IN S AND receiving_cell IN S`; +- filtering **edges** thins the tissue graph, because both layers read one array; +- the tissue graph is density-sampled by the same slider as the edge data. + +LRM filtering is already correctly decoupled — the tissue graph reads `lrm_count` +while the edge layer reads `visible_lrm_count`, so hiding mechanisms leaves the +structural graph intact. That is the pattern the other two filters should follow. + +## Design + +### The tissue graph gets its own fetch + +Not a flag on the shared result, and the reason is not obvious. The two layers +want *opposite* things from sampling: + +- the tissue graph wants **all** edges, then sampled for drawing volume; +- the edge data wants **filtered first, then sampled**, so that narrowing to a + rare subset draws that subset at full density rather than a tenth of it. That + ordering is the whole point of resolving filters server-side (issue #45). + +One query cannot do both, because density runs last (above). Returning every edge +with a `passes_filter` boolean would sample the rare subset away along with +everything else, quietly undoing #45 — the flag would be evaluated *after* the +sample had already thinned the rows it applies to. So: two requests, each keeping +filter-then-sample internally, with separate density controls. + +Sizing, measured across the bundled datasets — unique directed edges: + +| dataset | rows | unique edges | file | +|---|---|---|---| +| cosmx-mousebrain | 3,819,698 | 169,219 | 31 MB | +| visium_hd_tiny | 66,002 | 66,001 | 0.8 MB | +| xenium_human_breast_2fov | 134,277 | 44,759 | 3.2 MB | +| merscope-vpt-smallset | 21,587 | 14,503 | 0.3 MB | + +Rendering 169K–300K line segments is not the constraint; deck.gl handles that. +The payload is — `query-grouped` returns ~15 fields per edge, so a whole-tissue +view of a real run is tens of MB. The tissue graph therefore keeps a density +slider of its own, and the existing 500K-row cap stays as the backstop. A +**lean projection** for this endpoint (`x1,y1,x2,y2` and nothing else) is worth +doing at the same time: the structural layer draws lines and needs no scores, +types or LRM counts, which is most of the payload. + +### Edge filters become a list, and gain two cell-derived slots + +`edgeFilter` becomes `edgeFilters: MetadataFilter[]`, and-ed together. That alone +closes #45's deferred half and makes "sending = X and receiving = Y" expressible +against `edges.parquet` columns. + +For predicates on cell metadata that is *not* in `edges.parquet` — anything from +`cell-metadata/`, or a column the R export did not copy — two new spec slots +resolve through the existing cell path: + +``` +sendingFilter : MetadataFilter | null → filter_cell_ids() → id set S_send +receivingFilter : MetadataFilter | null → filter_cell_ids() → id set S_recv +``` + +`SpatialDatasetReader.filter_cell_ids(spec)` already resolves a cell-metadata +predicate to an id set and caches per (reader, spec). `duck.register_ids()` +already turns a large set into a hash semi-join rather than an unusable +`IN (?, ?, …)` list. The only change in `edge_reader.py` is to stop applying one +set to both ends: + +```python +# now +pred = duck.register_ids(conn, cell_ids, name="tp_cell_filter") +f'CAST("sending_cell" AS VARCHAR) {pred} AND CAST("receiving_cell" AS VARCHAR) {pred}' + +# after — two frames, two predicates, either side optional +send = duck.register_ids(conn, sending_ids, name="tp_send_filter") +recv = duck.register_ids(conn, receiving_ids, name="tp_recv_filter") +``` + +**Answering the issue's efficiency question directly:** this is the same cost as +today. One hash semi-join becomes two, on a query that already performs one, and +both run in the WHERE clause before the GROUP BY and before sampling. + +### Cell filtering stops touching edges + +`cellFilter` governs the cell layers only. The `cell_ids` argument stays on +`query_grouped` — the new sending/receiving slots use it — but the frontend +stops passing the cell-layer filter into the edge request. + +**This reverses a decision recorded in CLAUDE.md**, which argued that an edge +should be drawn only when both endpoints are, since "a half-outside edge would +run off to a cell that is not drawn". The lab's position is that edge structure +is worth seeing independently of which cells are rendered, so edges may now +terminate on cells that are not drawn. That is intended, not an oversight, and +the docs need correcting rather than the behaviour. + +## Consequences to get right + +- **Autocrine edges** have `sending_cell == receiving_cell`, so an autocrine edge + survives only when the cell satisfies *both* sides. Correct, but it means + setting sending and receiving to disjoint groups hides every autocrine ring. + Worth a note in the manual rather than a special case in code. +- **Direction is meaningless for the tissue graph**, which is undirected and now + unfiltered anyway — so the question does not arise. This is the main reason + directive 1 simplifies the feature rather than complicating it. +- **An empty id set means "nothing matches"**, and `register_ids` refuses an + empty frame. `query_grouped` already short-circuits `cell_ids == set()` to + `FALSE`; both new slots need the same guard, or a filter matching no cells + silently returns every edge. +- **`edgeFilter` → `edgeFilters` is a store migration.** Anything persisted or + copied between panels (`sanitiseSettings`) must handle both shapes, and the + push guard must validate every filter in the list, not just the first. + +## Surface + +| | | +|---|---| +| `backend/app/readers/edge_reader.py` | split the cell-id predicate; accept a filter list | +| `backend/app/routers/edges.py` | request model gains `sending_filter`, `receiving_filter`, `edge_filters`; new lean structural endpoint | +| `frontend/src/hooks/useEdges.js` | second unfiltered fetch for the graph; stop sending `cell_filter` | +| `frontend/src/components/Viewer.jsx` | tissue-graph layer reads the new array | +| `frontend/src/components/LayerPanel.jsx` | sending/receiving pickers; own density slider for the graph | +| `frontend/src/store.js` | `edgeFilters` list, `sendingFilter`, `receivingFilter`, `tissueGraphDensity` | + +`MetadataFilterSection` is already generic over a field list and a value fetcher, +so the two new pickers reuse it against the *cell* schema rather than the edge +schema. + +## Staging + +**1 — decouple, no new features.** Tissue graph gets its own unfiltered fetch and +density; `cellFilter` stops reaching the edge request. Directives 1 and 2, with +no change to what can be expressed. Independently useful and independently +reviewable. + +**2 — `edgeFilter` becomes a list.** Closes #45's deferred half; makes +sending/receiving expressible wherever the columns exist in `edges.parquet`. + +**3 — cell-derived sending/receiving slots.** The two id sets, for metadata that +is not in the edge file. + +**4 — lean structural projection** for the tissue-graph endpoint, if the payload +proves to be the limit on real data. Measure before doing it. + +**5 — docs.** CLAUDE.md's "cellFilter also governs edges" statement is wrong +after stage 1, in the store section and the metadata-subsetting section. The +manual's filter section needs the same correction, plus the autocrine note. + +## Testing + +Frontend store tests exist now (Vitest, 46), and the filter-targeting logic is +plain reducer code — the `edgeFilters` migration and the push-guard change belong +there. On the backend, the decisive check is a query-level one: for a dataset with +populated `sending_type`, assert that filtering sending alone leaves the receiving +distribution untouched (verified by hand above: 55 edges, one sending type, all +five receiving types), and that the tissue-graph endpoint returns an identical +count with and without every filter applied. From 3ffdc44e79dbe9ef94505d481f28669ba3390cf7 Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Tue, 18 Aug 2026 08:20:18 -0400 Subject: [PATCH 09/15] docs: correct the sending_type provenance, and revise the #59 plan around it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I claimed "half of #59 already ships", demonstrating a sending_type filter returning 55 of 223 edges on mouse_ileum_tiny. The mechanism works, but the demonstration was against invented data and the claim was wrong in substance. Tracing the column: - sample_data/make_edges.py writes it, and its own docstring says `cell type (simulated)` — the tidy Endothelial/Immune/Fibroblast labels in every bundled fixture are the generator's invention. - Of the six export scripts in r/, only niches_xenium.R can populate it, and only when the user passes --celltype. niches_cosmx, merscope, seqfish, visium and visium_hd all pass celltype.col = NULL, so on those platforms the column does not exist. So on real output it is absent five times out of six, opt-in on the sixth, and carries one label where the issue asks for any cell metadata column — mouse_ileum_tiny's cells table has cluster, region, pseudotime and seurat_clusters, none of which is in the edge file. The plan now resolves both dropdowns through the cells table uniformly, with no fast path: a fast path for the one case where the column happens to exist would mean two code paths answering the same question differently depending on which ran, and the frozen-at-scoring-time value can disagree with a re-annotated cells table. The testing section no longer leans on the simulated column either. Whether the column should exist at all is raised as an open question rather than answered — it is a NICHESv2-side decision, and whether it is duplication or provenance depends on whether NICHESv2 used the label when scoring, which is not answerable from this repo. docs/data_format.md and CLAUDE.md both presented it as a plain optional field, which reads as "usually there". Both now say what populates it and that it should not be relied on. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 12 ++- docs/data_format.md | 15 +++- docs/edge_filter_independence.md | 128 +++++++++++++++++++++++-------- 3 files changed, 119 insertions(+), 36 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index accce1b..f321e7c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -240,8 +240,16 @@ Platform-agnostic — works with any spatial dataset as long as cell barcodes ma | `score_norm` | float | Score normalized within edge (sums to 1) | | `x1`, `y1` | float | Sending cell centroid, native µm coords | | `x2`, `y2` | float | Receiving cell centroid | -| `sending_type` | string | Optional cell type label | -| `receiving_type` | string | Optional cell type label | +| `sending_type` | string | Optional cell type label — **often absent; see below** | +| `receiving_type` | string | Optional cell type label — **often absent; see below** | + +**`sending_type` / `receiving_type` cannot be relied on.** NICHESv2 populates them +only when given a `celltype.col`, and only `r/niches_xenium.R` exposes that flag +(`--celltype`) — the other five scripts pass `celltype.col = NULL`, so the columns +are simply not there. The values in `sample_data` are *simulated* by +`make_edges.py`. They are also frozen at scoring time and can disagree with a cells +table re-annotated since. Resolve cell attributes per edge against the cells table +(`filter_cell_ids`), not these columns. **Important**: Coordinates in edges.parquet are in native µm. The backend divides by `pixel_size` (from the reader) when serving to the frontend. diff --git a/docs/data_format.md b/docs/data_format.md index 4f27b7b..4b7157a 100644 --- a/docs/data_format.md +++ b/docs/data_format.md @@ -62,8 +62,19 @@ TissuePlex discovers `edges.parquet` automatically — no configuration needed. | Column | Type | Description | |--------|------|-------------| -| `sending_type` | string | Cell/spot type label for the sending unit | -| `receiving_type` | string | Cell/spot type label for the receiving unit | +| `sending_type` | string | Cell/spot type label for the sending unit — **optional, often absent** | +| `receiving_type` | string | Cell/spot type label for the receiving unit — **optional, often absent** | + +**Do not rely on `sending_type` / `receiving_type`.** `export_to_TissuePlex()` +populates them only when given a `celltype.col`, and of the scripts in `r/` only +`niches_xenium.R` exposes that (`--celltype`); the other five pass +`celltype.col = NULL`, so the columns are absent. The values in the bundled +`sample_data` fixtures are *simulated* by `make_edges.py`, not analysis output. + +They are also frozen at scoring time, so they can disagree with a cells table +that has since been re-annotated through `cell-metadata/`. Anything that needs a +cell attribute per edge should resolve it against the cells table instead — see +`docs/edge_filter_independence.md`. `export_to_TissuePlex()` **always writes these two columns**, so a file it produces has exactly the 16 columns above, in that order. Their *values* are `NA` when diff --git a/docs/edge_filter_independence.md b/docs/edge_filter_independence.md index ebbaa54..8194c8a 100644 --- a/docs/edge_filter_independence.md +++ b/docs/edge_filter_independence.md @@ -22,22 +22,52 @@ change current behaviour: --- -## What already works +## What already works, and why it does not count -Half of #59 ships today and is worth knowing before building anything. -`sending_type` and `receiving_type` are real populated columns in -`edges.parquet`, they are not in `EDGE_FILTER_SKIP`, so the edge filter dropdown -already offers them. Measured on `mouse_ileum_tiny`: +`sending_type` / `receiving_type` are in `edges.parquet` and are not in +`EDGE_FILTER_SKIP`, so the edge filter dropdown already offers them. Filtering +one side works mechanically — measured on `mouse_ileum_tiny`: | filter | edges | sending types in result | receiving types in result | |---|---|---|---| | none | 223 | all five | all five | | `sending_type = Fibroblast` | 55 | `Fibroblast` only | all five | -So one-sided filtering works. **The blocker is that `edgeFilter` holds exactly -one filter** — the composition gap deliberately deferred in issue #45 — so -"sending = Fibroblast *and* receiving = Endothelial" cannot be expressed. That -limit, not the sending/receiving distinction, is what needs fixing. +**This is not the feature, and the demo above is misleading.** Tracing where +those labels come from: + +| source | populates the column? | +|---|---| +| `sample_data/make_edges.py` | yes — its own docstring says `cell type (simulated)` | +| `r/niches_xenium.R` | only with an explicit `--celltype ` flag | +| `niches_cosmx/merscope/seqfish/visium/visium_hd.R` | **no** — all pass `celltype.col = NULL` | + +So the five tidy labels above (`Endothelial`, `Immune`, `Fibroblast`, …) are +**invented by the fixture generator**. On real output the column is absent on +five of six platforms as the export scripts are written today, and present on +Xenium only if the user opted in. It also carries exactly one column, where the +ask is "any cell metadata column" — `mouse_ileum_tiny`'s cells table has +`cluster`, `region`, `pseudotime` and `seurat_clusters`, none of which appears in +the edge file. + +The conclusion is not "half of #59 ships" — it is that the mechanism exists but +is pointed at the wrong source. The real blocker is twofold: `edgeFilter` holds +exactly one filter (the composition gap deferred in #45), and the only column it +can reach is one that mostly is not there. + +### Should `sending_type` exist at all? + +Worth deciding separately, and it is a NICHESv2-side question more than a +TissuePlex one. The case against: it duplicates cell metadata into the edge +table, freezing it at scoring time, and creates two sources of truth for the same +question — re-annotate cells through `cell-metadata/` and the edge file still says +the old thing. The case for: if NICHESv2 *used* the label when scoring, then it is +provenance rather than duplication, and the honest record of what produced the +number. Which of those is true is not answerable from this repo. + +Either way it should not back the two dropdowns. If it stays, document it as +"the label used at scoring time — may be absent, may be stale", and leave it +reachable through the edge-column filter where that framing is visible. ## Directive 3 already holds — preserve it @@ -119,29 +149,51 @@ slider of its own, and the existing 500K-row cap stays as the backstop. A doing at the same time: the structural layer draws lines and needs no scores, types or LRM counts, which is most of the payload. -### Edge filters become a list, and gain two cell-derived slots +### Two cell-metadata pickers, sending and receiving -`edgeFilter` becomes `edgeFilters: MetadataFilter[]`, and-ed together. That alone -closes #45's deferred half and makes "sending = X and receiving = Y" expressible -against `edges.parquet` columns. - -For predicates on cell metadata that is *not* in `edges.parquet` — anything from -`cell-metadata/`, or a column the R export did not copy — two new spec slots -resolve through the existing cell path: +This is the feature, not an add-on to an edge-column filter. The edge section +gets **two dropdowns side by side** — *sending cell* and *receiving cell* — and +each selects **any cell metadata column**, the same vocabulary the cell filter +offers. An edge is drawn when **both** sides are satisfied: the intersection. ``` -sendingFilter : MetadataFilter | null → filter_cell_ids() → id set S_send -receivingFilter : MetadataFilter | null → filter_cell_ids() → id set S_recv +Sending cell Receiving cell +[ region ▾ ] [ cluster ▾ ] +[x] crypt [x] 3 +[ ] mid [ ] 4 +[ ] villus [x] 7 ``` -`SpatialDatasetReader.filter_cell_ids(spec)` already resolves a cell-metadata -predicate to an id set and caches per (reader, spec). `duck.register_ids()` -already turns a large set into a hash semi-join rather than an unusable -`IN (?, ?, …)` list. The only change in `edge_reader.py` is to stop applying one -set to both ends: +Either side may be left unset, which leaves that end unconstrained — so one +dropdown alone answers "everything sent *from* crypt cells, to anywhere". + +### Resolve through the cells table, never the edge file's own labels + +Filtering on the edge file's own `sending_type` would be a plain SQL predicate, +cheaper than resolving an id set. It is still the wrong source, for three reasons +set out above: the column is **absent** on five of six platforms as the export +scripts stand, it carries **one** label where the ask is any column, and it is +**frozen** at scoring time so it can silently disagree with a re-annotated cells +table. + +A fast path for the one case where it happens to be present would mean two code +paths answering the same question differently depending on which one ran — the +kind of disagreement that is very hard to notice in a figure. + +So both dropdowns resolve through `filter_cell_ids()` against the cells table, +uniformly, with no fast path. The parquet's own labels stay reachable through the +separate edge-column filter, where they are honestly labelled as the as-scored +values. + +### The backend change + +`filter_cell_ids(spec)` already resolves a cell-metadata predicate to an id set +and caches per (reader, spec). `duck.register_ids()` already turns a large set +into a hash semi-join rather than an unusable `IN (?, ?, …)` list. The only change +in `edge_reader.py` is to stop applying one set to both ends: ```python -# now +# now — one set, both endpoints pred = duck.register_ids(conn, cell_ids, name="tp_cell_filter") f'CAST("sending_cell" AS VARCHAR) {pred} AND CAST("receiving_cell" AS VARCHAR) {pred}' @@ -150,9 +202,20 @@ send = duck.register_ids(conn, sending_ids, name="tp_send_filter") recv = duck.register_ids(conn, receiving_ids, name="tp_recv_filter") ``` +Both predicates go in the same `WHERE` clause, so they run before the `GROUP BY` +and before the sample — the ordering directive 3 requires. + **Answering the issue's efficiency question directly:** this is the same cost as -today. One hash semi-join becomes two, on a query that already performs one, and -both run in the WHERE clause before the GROUP BY and before sampling. +today. One hash semi-join becomes two, on a query that already performs one. + +### Edge-column filters stay, and become a list + +Separately from the two pickers, `edgeFilter` becomes `edgeFilters: +MetadataFilter[]`, and-ed together. This is what `edge-metadata/` annotations +(a confidence, a review flag) and the parquet's own columns are filtered through, +and it closes the composition gap deferred in #45. It is a smaller, independent +win — the two cell-metadata pickers are what #59 actually asks for. + ### Cell filtering stops touching edges @@ -223,8 +286,9 @@ manual's filter section needs the same correction, plus the autocrine note. Frontend store tests exist now (Vitest, 46), and the filter-targeting logic is plain reducer code — the `edgeFilters` migration and the push-guard change belong -there. On the backend, the decisive check is a query-level one: for a dataset with -populated `sending_type`, assert that filtering sending alone leaves the receiving -distribution untouched (verified by hand above: 55 edges, one sending type, all -five receiving types), and that the tissue-graph endpoint returns an identical -count with and without every filter applied. +there. On the backend the decisive checks are query-level, and should use a **cell +metadata column** rather than `sending_type`, which the fixture only simulates: +filtering sending alone must leave the receiving distribution untouched; the two +sides together must return the intersection; and the tissue-graph endpoint must +return an identical count with and without every filter applied, which is +directive 1 stated as an assertion. From 6638cc68e132020917f3ccf1c8e2a35370d964ad Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Tue, 18 Aug 2026 08:54:50 -0400 Subject: [PATCH 10/15] feat: independent sending/receiving edge filters, and unfilter the tissue graph (#59) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements docs/edge_filter_independence.md. The pipeline is now explicit, and the order is the contract: all edges in viewport → density filter deterministic, spatially random → EDGESET A → tissue-graph layer (query_structure) → sending filter → receiving filter → edge-table filters → EDGESET B → edge-data layer (query_grouped) Three lab directives drove it, two of which reversed existing behaviour. The tissue graph is ground truth. It gets its own query, and query_structure takes no filter arguments at all — not "they default to none", but no parameter to pass, so nothing can wire one in later. It is a separate request rather than a flag on the shared one because the two layers want opposite things from sampling, and a passes_filter column would be read after the sample had already thinned the rows it describes. The projection is lean (edge + four coordinates), measured 2.2–2.4x smaller than the grouped payload. Cell and edge filtering are independent. cell_filter no longer reaches the edge request at all; sending_ids and receiving_ids constrain the two endpoints separately, so both set gives the intersection and one set leaves the other end free. An edge may now terminate on a cell that is not drawn, which is intended. Both resolve from the *cells table* rather than the edge file's own sending_type/receiving_type — those are absent on five of six platforms as the r/ scripts stand, carry one label where any cell column is wanted, and are frozen at scoring time. Density is one slider over both layers. A second slider was built and removed: the argument for it does not hold, since unfiltered the two layers draw exactly the same number of lines and the graph already has an opacity control, which is the better lever for clutter. The real bug in the first cut was that the two queries sampled independently. Two bernoulli draws at 10% overlap only ~1%, so edge data appeared where the graph beneath it had been sampled away. density_predicate replaces USING SAMPLE with a deterministic hash of the edge id: the same edge gets the same verdict in every query, so the predicate commutes with the filters and B is a subset of A at every density. It is also stable across re-fetches, where bernoulli flickered on each pan. Also: edge_filter becomes edge_filters, a list and-ed together, closing the composition gap deferred in #45. The old cell_filter and edge_filter fields are still accepted so an older frontend against a newer backend keeps working. Verified rather than assumed: - The intersection matches ground truth computed independently from the parquet and the metadata CSV — a full 3x3 sending/receiving matrix, zero mismatches. - backend/tests/edge_pipeline_check.py asserts the graph is unmoved by any filter, that edge data is always a subset of it, and that sampling is stable: 9 datasets across 6 platforms, at densities 1.0 / 0.5 / 0.1. - Visually on Xenium: applying a sending filter leaves the grey graph pixel-for- pixel identical while the coloured edge layer thins; at 15% density every coloured edge sits on a grey line. - Split screen with two datasets and per-panel endpoint filters, clean console. - Golden snapshot 249 probes, frontend 52 tests, duckdb config check. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 73 +++++++--- backend/app/main.py | 2 +- backend/app/readers/edge_reader.py | 191 ++++++++++++++++++++----- backend/app/routers/edges.py | 65 ++++++++- backend/tests/edge_pipeline_check.py | 114 +++++++++++++++ backend/tests/golden_baseline.json | 121 ++++++++++++++++ backend/tests/golden_snapshot.py | 7 +- docs/edge_filter_independence.md | 47 ++++-- docs/index.html | 70 +++++++-- frontend/package.json | 2 +- frontend/src/components/LayerPanel.jsx | 97 ++++++++++++- frontend/src/components/Viewer.jsx | 19 ++- frontend/src/hooks/useEdges.js | 74 ++++++++-- frontend/src/store.js | 39 ++++- frontend/src/store.settings.test.js | 69 ++++++++- 15 files changed, 876 insertions(+), 114 deletions(-) create mode 100644 backend/tests/edge_pipeline_check.py diff --git a/CLAUDE.md b/CLAUDE.md index f321e7c..89a0501 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -416,10 +416,16 @@ All shared state lives in a single Zustand store. Key sections: panel 0 — the LayerPanel reads these instead of guessing from the schema dtype. - **Categorical override**: `categoricalOverrides`, keyed `cell::` / `edge::` → `true | false`; absent means auto-detect (issue #35). -- **Metadata filter**: `cellFilter` / `edgeFilter`, each - `{ field, values, min, max, includeMissing }` or null (issue #45). `cellFilter` - also governs edges — both endpoints must survive it. Both reset on dataset change - (column names are dataset-specific); `edgeFilter` also resets on edge-file change. +- **Metadata filters**: each is `{ field, values, min, max, includeMissing }` or null. + - `cellFilter` — the cell layers only. It no longer governs edges (issue #59): + filtering cells and filtering edges are independent actions. + - `sendingFilter` / `receivingFilter` — a *cell* metadata predicate on one end of + an edge. Both set gives the intersection; one set leaves the other end free. + - `edgeFilters` — a **list**, and-ed, on the edge table / `edge-metadata/`. + + All reset on dataset change (column names are dataset-specific); `edgeFilters` + also resets on edge-file change, since those column names belong to one file + while cell metadata does not. - **Transcript gene filter**: `selectedGenes` — `null` = no filter (show all); `Set` = allowlist (show only those genes). Dataset-scoped; resets on dataset change. See Gene Filter section below. @@ -1127,6 +1133,15 @@ Things to preserve when editing these methods: the same rows or the layer visibly flickers. - **`USING SAMPLE` goes on a subquery** wrapping the filtered SELECT. Applied alongside a WHERE clause, DuckDB may sample before filtering. +- **Edge density is a deterministic hash, not `USING SAMPLE`** (`density_predicate`). + The tissue graph and the edge data are two separate queries and must select the + *same* edges, or edge data is drawn where the graph beneath it was sampled away — + two independent bernoulli draws at 10% overlap only ~1% of the time. Hashing the + edge id gives every edge the same verdict in every query, so the predicate commutes + with the filters (density-then-filter and filter-then-density are one set) and + B ⊆ A holds at every density. It is also stable across re-fetches, where bernoulli + flickers on each pan. `backend/tests/edge_pipeline_check.py` asserts this on every + dataset. - **DuckDB cannot bind numpy scalars.** `bbox_predicate()` casts to builtin `float` for this reason. - A fresh `connect()` per call is deliberate — DuckDB's global connection is not @@ -1227,19 +1242,31 @@ subset renders at full density. count and the sample**. The five implementations differ too much to share code: Xenium and CosMx join it into their DuckDB query, MERSCOPE skips non-matching rows before decoding WKB, Visium HD and seqFISH mask their in-memory frames. -- `EdgeReader.query_grouped()` takes `cell_ids` and `edge_filter`. An edge survives - the cell filter only when **both** endpoints do — the point of "focus on 2–3 cell - types" is the signalling within that subset, and a half-outside edge would run off - to a cell that is not drawn. `edge_filter` becomes a real SQL predicate when the - column is in the parquet, and a semi-join against a registered frame when it comes - from `edge-metadata/`. - - **The both-endpoints rule is slated to be reversed.** The lab's position is that - cell and edge filtration should be completely independent, and that the tissue - graph is ground truth that filtering must never subset — today one `useEdges` - request feeds both the graph and the edge layer, so both couplings are live. See - `docs/edge_filter_independence.md` (issue #59). The reasoning above is the decision - being reversed, kept because it explains what the code currently does. +- **`EdgeReader` runs one pipeline, and the order is the contract** (issue #59): + + ``` + all edges in viewport + → density filter deterministic, spatially random + → EDGESET A → tissue-graph layer (query_structure) + → sending filter + → receiving filter + → edge-table filters + → EDGESET B → edge-data layer (query_grouped) + ``` + + `query_structure` takes **no filter arguments at all** — not "they default to + none", but no parameter to pass — because the tissue graph is ground truth: the + total set of edges, shown or hidden, never subset. + + `sending_ids` / `receiving_ids` constrain the two endpoints independently, so both + set gives the intersection and one set leaves the other end free. They resolve from + *cell* metadata via `filter_cell_ids`, and are independent of the cell layer's own + filter — an edge may terminate on a cell that is not drawn. That reverses an earlier + both-endpoints rule: filtering cells and filtering edges are separate actions. + + `edge_filters` is a **list**, and-ed — the composition gap deferred in #45. Each + becomes a real SQL predicate when the column is in the parquet, and a semi-join + against a registered frame when it comes from `edge-metadata/`. **Large id sets go through `duck.register_ids()`, not `IN (?, ?, …)`.** A filter can keep hundreds of thousands of cells; binding that many parameters is unworkable and @@ -1371,6 +1398,18 @@ cd frontend && npm test # vitest run cd frontend && npm run test:watch ``` +Two further backend checks exist alongside the golden snapshot, both covering +things it structurally cannot: + +```bash +cd backend && python3 tests/edge_pipeline_check.py # issue #59 invariants +cd backend && python3 tests/duckdb_config_check.py # DUCKDB_MEMORY_LIMIT forms +``` + +`edge_pipeline_check` asserts, on every dataset with edges and at several +densities, that the tissue graph is unmoved by any filter, that the edge data is +always a subset of it, and that sampling is stable between identical calls. + `src/store.annotations.test.js` is the first of them. Store logic is plain JS, so these need no DOM and no jsdom dependency — reducers can be exercised directly through `useStore.getState()`. Coverage is currently annotations only. diff --git a/backend/app/main.py b/backend/app/main.py index 26a8700..776a1f3 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -3,7 +3,7 @@ from app.routers import tiles, spatial, edges, layers -APP_VERSION = "0.8.5" +APP_VERSION = "0.8.6" app = FastAPI(title="TissuePlex API", version=APP_VERSION) diff --git a/backend/app/readers/edge_reader.py b/backend/app/readers/edge_reader.py index 57a29b0..2ee810e 100644 --- a/backend/app/readers/edge_reader.py +++ b/backend/app/readers/edge_reader.py @@ -27,6 +27,53 @@ _UNSET = object() +# The pipeline, in one place: +# +# all edges in viewport +# → density filter (spatially random, this predicate) +# → EDGESET A → tissue-graph layer +# → sending filter +# → receiving filter +# → edge-table filters +# → EDGESET B → edge-data layer +# +# So B ⊆ A always: edge data can never be drawn where the graph beneath it has +# been sampled away. +# +# The two layers are fetched by separate queries, and the order above is only +# guaranteed because this predicate is *deterministic per edge*. A given edge +# gets the same verdict in every query regardless of what else ran, so the +# predicate commutes with the filters and "density then filter" and "filter then +# density" select the identical set. +# +# `USING SAMPLE` cannot do this. Two independent bernoulli draws at 10% give the +# graph one random tenth and the edge layer a different one — an overlap of ~1%, +# with edge data floating free of the structure it is supposed to sit on. +# +# A deterministic hash of the edge identity fixes that. The same edge gets the +# same verdict in every query, whatever filters ran, so the edge layer is always +# a subset of the graph: +# +# graph = { e : keep(e) } +# edge data = { e : passes_filters(e) and keep(e) } ⊆ graph +# +# It is also spatially uniform (the hash ignores position) and stable across +# re-fetches, which matters because the alternative flickers on every pan. +# +# Unlike USING SAMPLE this is an ordinary predicate, so it composes with the +# filters by AND and cannot be reordered below them — the property the +# subquery-wrapping was there to guarantee. +_DENSITY_MODULUS = 1_000_000 + + +def density_predicate(density: float, key_sql: str = '"edge"') -> str: + """SQL keeping a deterministic, spatially uniform fraction of edges.""" + if density is None or density >= 1.0: + return "" + cut = max(1, int(density * _DENSITY_MODULUS)) + return f"(hash(CAST({key_sql} AS VARCHAR)) % {_DENSITY_MODULUS}) < {cut}" + + class EdgeReader: def __init__(self, path: Path, pixel_size: float = 1.0): self.path = path @@ -273,21 +320,42 @@ def edge_filter_sql(self, spec: Optional[MetadataFilter], conn) -> tuple[str, li return f'CAST("edge" AS VARCHAR) {pred}', [] @staticmethod - def cell_filter_sql(cell_ids: Optional[set], conn) -> tuple[str, list]: - """WHERE fragment keeping only edges whose **both** endpoints survive the - cell filter. - - Both, not either: the point of "focus on 2–3 cell types" is the signalling - *within* that subset. An edge with one endpoint outside would be drawn - running off to a cell that is not on screen. + def endpoint_filter_sql( + sending_ids: Optional[set], + receiving_ids: Optional[set], + conn, + ) -> tuple[str, list]: + """WHERE fragment constraining each endpoint of an edge independently. + + Either side may be None, which leaves that end unconstrained — so setting + only `sending_ids` answers "everything sent *from* these cells, to + anywhere". With both set the result is the intersection: an edge is kept + when its sender is in one set and its receiver in the other. + + This replaces a single set applied to both ends. That older rule tied edge + visibility to the *cell* filter, which the lab wants independent: filtering + cells is one action, filtering edges another, and an edge may now terminate + on a cell that is not drawn. See docs/edge_filter_independence.md. + + An empty set means "nothing matches" and short-circuits to FALSE. It must + not fall through to no-predicate, or a filter matching no cells would + return every edge — and `register_ids` refuses an empty frame anyway. + + Both predicates are ordinary semi-joins in the WHERE clause, so they run + before the GROUP BY and before the density sample. """ - if cell_ids is None: - return "", [] - if not cell_ids: - return "FALSE", [] - pred = duck.register_ids(conn, cell_ids, name="tp_cell_filter") - return (f'CAST("sending_cell" AS VARCHAR) {pred} ' - f'AND CAST("receiving_cell" AS VARCHAR) {pred}'), [] + conds: list[str] = [] + for ids, col, name in ( + (sending_ids, "sending_cell", "tp_send_filter"), + (receiving_ids, "receiving_cell", "tp_recv_filter"), + ): + if ids is None: + continue + if not ids: + return "FALSE", [] + pred = duck.register_ids(conn, ids, name=name) + conds.append(f'CAST("{col}" AS VARCHAR) {pred}') + return (" AND ".join(conds), []) if conds else ("", []) def edge_detail(self, edge_id: str) -> dict | None: """Return all LRM rows for a single directed edge, structured for the info panel.""" @@ -357,14 +425,74 @@ def column_summary(self, column: str) -> dict: ).df().iloc[:, 0].tolist() return {"type": "categorical", "values": vals, "count": len(vals)} + def query_structure( + self, + bbox: Optional[tuple] = None, + density: float = 1.0, + max_limit: int = 500_000, + ) -> list[dict]: + """Every distinct edge in the viewport, with no filters of any kind. + + This backs the tissue-graph layer, which is *ground truth*: the total set + of edges, shown or hidden, never subset. It deliberately takes no filter + arguments at all — not "filters default to none", but no way to pass one, + so the layer cannot be narrowed by a future caller wiring one through. + + It is a separate query from `query_grouped`, not a flag on it, because the + two want opposite orderings of the same pipeline. The edge-data layer must + filter *then* sample, so a rare subset draws at full density; the graph + must not filter at all. Returning every edge with a `passes_filter` column + would sample the rare subset away before the flag was ever read. + + The projection is deliberately lean — `edge` and the four coordinates. + Scores, types and LRM counts are most of `query_grouped`'s payload and the + structural layer draws none of them; `edge` is kept only because the layer + is pickable and the info panel resolves by id. + """ + cols = set(self._parquet_schema().names) + if not {"x1", "y1", "x2", "y2"} <= cols: + return [] + ps = self.pixel_size + + where, params = "", [] + if bbox and None not in bbox: + xmin, ymin, xmax, ymax = (v * ps for v in bbox) + where = ("WHERE ((x1 >= ? AND x1 <= ? AND y1 >= ? AND y1 <= ?) OR " + "(x2 >= ? AND x2 <= ? AND y2 >= ? AND y2 <= ?))") + params = [xmin, xmax, ymin, ymax, xmin, xmax, ymin, ymax] + + has_edge = "edge" in cols + key = "edge" if has_edge else "x1, y1, x2, y2" + sel = ("edge, " if has_edge else "") + \ + "FIRST(x1) AS x1, FIRST(y1) AS y1, FIRST(x2) AS x2, FIRST(y2) AS y2" + # The *same* predicate query_grouped uses, on the same key, so the two + # layers select an identical subset and edge data is never drawn where + # the graph beneath it has been sampled away. + dens = density_predicate( + density, '"edge"' if has_edge else "concat_ws('|',x1,y1,x2,y2)") + sample = f"WHERE {dens}" if dens else "" + + with self._conn() as conn: + df = conn.execute(f""" + SELECT * FROM ( + SELECT {sel} FROM {self._from()} {where} GROUP BY {key} + ) {sample} + LIMIT {max_limit} + """, params).df() + + for c in ("x1", "y1", "x2", "y2"): + df[c] = df[c] / ps + return df.to_dict("records") + def query_grouped( self, bbox: Optional[tuple] = None, min_lrm_count: int = 1, density: float = 1.0, max_limit: int = 500_000, - cell_ids: Optional[set] = None, - edge_filter: Optional[MetadataFilter] = None, + sending_ids: Optional[set] = None, + receiving_ids: Optional[set] = None, + edge_filters: Optional[list] = None, ) -> list[dict]: """ Return one row per directed edge (GROUP BY edge), pre-aggregated. @@ -382,10 +510,13 @@ def query_grouped( density<1.0 uses bernoulli sampling so each edge is independently included with probability `density` — spatially uniform. - `cell_ids` and `edge_filter` are the metadata filters from issue #45. Both - go into the WHERE clause, so they run before the GROUP BY and before the - density sample: filtering to a rare cell type keeps that type's edges at - full density rather than sampling them away. + `sending_ids` / `receiving_ids` constrain the two endpoints independently + (issue #59); `edge_filters` is a list of MetadataFilter and-ed together, + which is the composition #45 deferred. All of them go into the WHERE + clause, so they run before the GROUP BY and before the density sample: + narrowing to a rare subset keeps it at full density rather than sampling + it away. Density is last, and deliberately so — it is a rendering-volume + control, not a selection criterion. """ ps = self.pixel_size schema_names = set(self._parquet_schema().names) @@ -428,13 +559,10 @@ def query_grouped( select = ", ".join(agg_cols) - # Bernoulli sampling: each grouped edge row is included independently - # at probability `density`. At density=1.0 no sampling clause is added - # and all viewport edges are returned (up to max_limit safety cap). - sample_clause = ( - f"USING SAMPLE {density * 100:.4f} PERCENT (bernoulli)" - if density < 1.0 else "" - ) + # Deterministic, shared with query_structure so the edge layer is always a + # subset of the tissue graph at the same density. See density_predicate. + dens = density_predicate(density) + sample_clause = f"WHERE {dens}" if dens else "" # The connection is opened before the WHERE clause is finalised because the # metadata filters may need to register a relation on it to semi-join @@ -444,13 +572,12 @@ def query_grouped( # An edge file without endpoint columns cannot be filtered by cell; the # tissue graph still draws, it just ignores the cell subset. if not {"sending_cell", "receiving_cell"} <= schema_names: - cell_ids = None + sending_ids = receiving_ids = None with self._conn() as conn: - for cond, prm in ( - self.cell_filter_sql(cell_ids, conn), - self.edge_filter_sql(edge_filter, conn), - ): + fragments = [self.endpoint_filter_sql(sending_ids, receiving_ids, conn)] + fragments += [self.edge_filter_sql(f, conn) for f in (edge_filters or [])] + for cond, prm in fragments: if cond: where_conditions.append(cond) where_params.extend(prm) diff --git a/backend/app/routers/edges.py b/backend/app/routers/edges.py index 58540af..c2d7f76 100644 --- a/backend/app/routers/edges.py +++ b/backend/app/routers/edges.py @@ -169,9 +169,27 @@ class EdgeGroupedQueryRequest(BaseModel): ymax: Optional[float] = None min_strength: Optional[float] = None density: float = 1.0 # fraction of viewport edges to return (0.01–1.0) - # cell_filter restricts by *cell* metadata: an edge survives only if both of - # its endpoints do. edge_filter restricts by a column of the edge table itself - # (or of edge-metadata/). They compose. + + # Endpoint filters (issue #59). Each is a *cell* metadata predicate resolved + # against the cells table and applied to one end of the edge, so the two + # compose as an intersection: sender in A, receiver in B. Either may be + # omitted, leaving that end unconstrained. + # + # These are resolved against the cells table rather than the edge file's own + # sending_type/receiving_type: those are absent on five of six platforms as + # the r/ export scripts stand, carry one label where any cell column is + # wanted, and are frozen at scoring time. See docs/edge_filter_independence.md. + sending_filter: Optional[MetadataFilterSpec] = None + receiving_filter: Optional[MetadataFilterSpec] = None + + # Filters on the edge table itself (or edge-metadata/), and-ed together. + # A list rather than one filter — the composition gap deferred in #45. + edge_filters: Optional[List[MetadataFilterSpec]] = None + + # Superseded. cell_filter applied one cell predicate to *both* endpoints and + # tied edge visibility to the cell layer's filter; cell and edge filtering are + # now independent actions. edge_filter is the pre-list singular form. Both are + # still accepted so an older frontend against a newer backend keeps working. cell_filter: Optional[MetadataFilterSpec] = None edge_filter: Optional[MetadataFilterSpec] = None @@ -194,6 +212,30 @@ def _cell_ids_for(dataset: str, spec: Optional[MetadataFilterSpec]) -> Optional[ raise HTTPException(400, str(exc)) +class EdgeStructureRequest(BaseModel): + xmin: Optional[float] = None + ymin: Optional[float] = None + xmax: Optional[float] = None + ymax: Optional[float] = None + density: float = 1.0 + # No filter fields, deliberately. The tissue graph is the total set of edges; + # see EdgeReader.query_structure. + + +@router.post("/{dataset}/query-structure") +def query_edge_structure(dataset: str, body: EdgeStructureRequest, + edge_file: str = Query("edges.parquet")): + """Every edge in the viewport, unfiltered — the tissue-graph layer. + + Separate from /query-grouped so that filtering the edge *data* can never + subset the structural graph, and so the two can be sampled independently. + """ + bbox = (body.xmin, body.ymin, body.xmax, body.ymax) \ + if body.xmin is not None else None + return _reader(dataset, edge_file).query_structure( + bbox=bbox, density=max(0.001, min(1.0, body.density))) + + @router.post("/{dataset}/query-grouped") def query_edges_grouped(dataset: str, body: EdgeGroupedQueryRequest, edge_file: str = Query("edges.parquet")): @@ -207,11 +249,24 @@ def query_edges_grouped(dataset: str, body: EdgeGroupedQueryRequest, if body.xmin is not None else None density = max(0.001, min(1.0, body.density)) try: + # Legacy cell_filter means "both endpoints", i.e. the same set on each. + legacy = _cell_ids_for(dataset, body.cell_filter) + sending = _cell_ids_for(dataset, body.sending_filter) + receiving = _cell_ids_for(dataset, body.receiving_filter) + if legacy is not None: + sending = legacy if sending is None else sending & legacy + receiving = legacy if receiving is None else receiving & legacy + + filters = [f.build() for f in (body.edge_filters or [])] + if body.edge_filter is not None: + filters.append(body.edge_filter.build()) + return _reader(dataset, edge_file).query_grouped( bbox=bbox, density=density, - cell_ids=_cell_ids_for(dataset, body.cell_filter), - edge_filter=body.edge_filter.build() if body.edge_filter else None, + sending_ids=sending, + receiving_ids=receiving, + edge_filters=[f for f in filters if f is not None], ) except ValueError as exc: raise HTTPException(400, str(exc)) diff --git a/backend/tests/edge_pipeline_check.py b/backend/tests/edge_pipeline_check.py new file mode 100644 index 0000000..3fafa6d --- /dev/null +++ b/backend/tests/edge_pipeline_check.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +""" +The edge pipeline, asserted (issue #59). + + all edges in viewport + → density filter (spatially random, deterministic per edge) + → EDGESET A → tissue-graph layer + → sending filter + → receiving filter + → edge-table filters + → EDGESET B → edge-data layer + +Three properties fall out, and all three are things a plausible refactor breaks +silently rather than loudly: + +1. **The tissue graph is ground truth.** Its count must not move no matter what + filters are set on the edge data. `query_structure` takes no filter arguments + at all, so this is really asserting that nobody wired one in. +2. **B is a subset of A, at every density.** Edge data must never be drawn where + the graph beneath it was sampled away. This is why density is a deterministic + hash of the edge id rather than `USING SAMPLE`: two independent bernoulli + draws at 10% overlap only ~1% of the time. +3. **Sampling is stable.** The same request twice returns the same edges, or the + layer flickers on every pan. + +Run against whatever datasets are present: + + python3 tests/edge_pipeline_check.py +""" +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +os.environ.setdefault("DATA_ROOT", str(Path(__file__).resolve().parents[2] / "sample_data")) + +from app.readers.metadata_filter import MetadataFilter # noqa: E402 +from app.routers import spatial, edges as edges_router # noqa: E402 + +DATA_ROOT = Path(os.environ["DATA_ROOT"]) +DENSITIES = (1.0, 0.5, 0.1) +failures: list[str] = [] + + +def usable_field(frame): + """A metadata column with real, varied values — not an all-NaN one.""" + for c in frame.columns: + if c in ("x_centroid", "y_centroid", "cell_id"): + continue + if frame[c].notna().any() and frame[c].nunique() > 1: + return c + return None + + +def spec_for(reader, field): + cv = reader.color_values("metadata", None, field) + if cv.get("categories"): + return MetadataFilter.build(field, values=cv["categories"][:1]) + lo, hi = cv.get("min"), cv.get("max") + if lo is None or hi is None or lo == hi: + return None + return MetadataFilter.build(field, vmin=lo + (hi - lo) * 0.4, vmax=hi) + + +def check(ds: str) -> str: + sr = spatial._reader(ds) + er = edges_router._reader(ds, "edges.parquet") + field = usable_field(sr._metadata_frame()) + spec = spec_for(sr, field) if field else None + ids = sr.filter_cell_ids(spec) if spec else None + + baseline = len(er.query_structure(density=1.0)) + for d in DENSITIES: + A = {r["edge"] for r in er.query_structure(density=d)} + B = {r["edge"] for r in er.query_grouped(density=d)} + if B != A: + failures.append(f"[{ds} d={d}] unfiltered edge data != graph " + f"({len(B)} vs {len(A)})") + if ids: + Bf = {r["edge"] for r in er.query_grouped(density=d, sending_ids=ids)} + if not Bf <= A: + failures.append(f"[{ds} d={d}] filtered edge data is not a subset " + f"of the graph ({len(Bf - A)} strays)") + # Property 1: the graph is unmoved by anything the edge data did. + if len(er.query_structure(density=1.0)) != baseline: + failures.append(f"[{ds} d={d}] tissue graph count moved") + + # Property 3 + if {r["edge"] for r in er.query_structure(density=0.1)} != \ + {r["edge"] for r in er.query_structure(density=0.1)}: + failures.append(f"[{ds}] sampling is not stable between identical calls") + + return f"{baseline:>8,} edges filter=[{field}]" if field else f"{baseline:>8,} edges (no filterable column)" + + +found = sorted(p.name for p in DATA_ROOT.iterdir() if (p / "edges.parquet").exists()) +if not found: + print("no edge datasets present — nothing to check") + sys.exit(0) + +for ds in found: + try: + print(f" OK {ds:26} {check(ds)}") + except Exception as exc: + failures.append(f"[{ds}] {type(exc).__name__}: {exc}") + print(f" FAIL {ds:26} {type(exc).__name__}: {exc}") + +print() +if failures: + for f in failures: + print(" ✗", f) + sys.exit(1) +print(f"edge pipeline holds across {len(found)} datasets " + f"at densities {', '.join(str(d) for d in DENSITIES)}.") diff --git a/backend/tests/golden_baseline.json b/backend/tests/golden_baseline.json index 87d87cb..55b2ec0 100644 --- a/backend/tests/golden_baseline.json +++ b/backend/tests/golden_baseline.json @@ -179,6 +179,17 @@ ], "n": 169219 }, + "edge__edges.parquet__structure": { + "digest": "c41185a78aa1dea2", + "keys": [ + "edge", + "x1", + "x2", + "y1", + "y2" + ], + "n": 169219 + }, "gene_list": { "digest": "b4a6b682f016cebd", "n": 960 @@ -391,6 +402,17 @@ ], "n": 14503 }, + "edge__edges.parquet__structure": { + "digest": "e5e502e5a4f78494", + "keys": [ + "edge", + "x1", + "x2", + "y1", + "y2" + ], + "n": 14503 + }, "gene_list": { "digest": "ba0a34a6d5631ace", "n": 130 @@ -626,6 +648,17 @@ ], "n": 223 }, + "edge__edges.parquet__structure": { + "digest": "0676dafca6cb1c65", + "keys": [ + "edge", + "x1", + "x2", + "y1", + "y2" + ], + "n": 223 + }, "edge__edges__edge.normalized.product.parquet__catalogue": { "digest": "556c70f23429ddd8", "n": 20 @@ -703,6 +736,17 @@ ], "n": 289 }, + "edge__edges__edge.normalized.product.parquet__structure": { + "digest": "a7f95b42286cb2bf", + "keys": [ + "edge", + "x1", + "x2", + "y1", + "y2" + ], + "n": 289 + }, "edge__edges__edge.raw.minimum.parquet__catalogue": { "digest": "556c70f23429ddd8", "n": 20 @@ -780,6 +824,17 @@ ], "n": 155 }, + "edge__edges__edge.raw.minimum.parquet__structure": { + "digest": "2cea4f58c308f7c1", + "keys": [ + "edge", + "x1", + "x2", + "y1", + "y2" + ], + "n": 155 + }, "filter_bounds__transcript_counts": { "n_cells": 19, "total": 19 @@ -1034,6 +1089,17 @@ ], "n": 372 }, + "edge__edges.parquet__structure": { + "digest": "dcebdff0bd9e434e", + "keys": [ + "edge", + "x1", + "x2", + "y1", + "y2" + ], + "n": 372 + }, "gene_list": { "digest": "6f1e76b0bd839528", "n": 12 @@ -1245,6 +1311,17 @@ ], "n": 250 }, + "edge__edges.parquet__structure": { + "digest": "1be5a577ef35b7a4", + "keys": [ + "edge", + "x1", + "x2", + "y1", + "y2" + ], + "n": 250 + }, "gene_list": { "digest": "66064248776a3e2f", "n": 36 @@ -1454,6 +1531,17 @@ ], "n": 66001 }, + "edge__edges.parquet__structure": { + "digest": "b51d25a17fca5fc2", + "keys": [ + "edge", + "x1", + "x2", + "y1", + "y2" + ], + "n": 66001 + }, "gene_list": { "digest": "40a2e076472f6c63", "n": 32245 @@ -1631,6 +1719,17 @@ ], "n": 1642 }, + "edge__edges.parquet__structure": { + "digest": "f8bfac32075532cd", + "keys": [ + "edge", + "x1", + "x2", + "y1", + "y2" + ], + "n": 1642 + }, "filter_bounds__array_row": { "n_cells": 18, "total": 18 @@ -1831,6 +1930,17 @@ ], "n": 44759 }, + "edge__edges.parquet__structure": { + "digest": "f9eacf53fffc98ca", + "keys": [ + "edge", + "x1", + "x2", + "y1", + "y2" + ], + "n": 44759 + }, "edge__edges__niches_rad30.parquet__catalogue": { "digest": "931e3a76e6dcb943", "n": 32 @@ -1908,6 +2018,17 @@ ], "n": 181523 }, + "edge__edges__niches_rad30.parquet__structure": { + "digest": "abc6dda554c2ad29", + "keys": [ + "edge", + "x1", + "x2", + "y1", + "y2" + ], + "n": 181523 + }, "filter_bounds__control_probe_counts": { "n_cells": 7272, "total": 7272 diff --git a/backend/tests/golden_snapshot.py b/backend/tests/golden_snapshot.py index ab3a347..527ae03 100644 --- a/backend/tests/golden_snapshot.py +++ b/backend/tests/golden_snapshot.py @@ -311,9 +311,14 @@ def probe_edges(dataset_dir: Path, pixel_size: float, p: Probes) -> None: continue spec = MetadataFilter.build(col, values=cv["categories"][:1]) p.record(f"edge__{key}__filter__{col}", lambda er=er, spec=spec: - Probes.rows(er.query_grouped(density=1.0, edge_filter=spec))) + Probes.rows(er.query_grouped(density=1.0, edge_filters=[spec]))) break + # Issue #59: the structural query is the tissue graph's ground truth and + # takes no filters, so its count must equal the unfiltered edge count. + p.record(f"edge__{key}__structure", + lambda er=er: Probes.rows(er.query_structure(density=1.0))) + def collect() -> dict: from app.readers.reader_factory import ReaderFactory diff --git a/docs/edge_filter_independence.md b/docs/edge_filter_independence.md index 8194c8a..b529690 100644 --- a/docs/edge_filter_independence.md +++ b/docs/edge_filter_independence.md @@ -1,6 +1,18 @@ # Issue #59 — independent sending / receiving edge filters -Status: **planned, not started.** +Status: **implemented** in v0.8.6. Kept as the design record. + +The pipeline as built: + +``` +all edges in viewport + → density filter deterministic, spatially random + → EDGESET A → tissue-graph layer (query_structure) + → sending filter + → receiving filter + → edge-table filters + → EDGESET B → edge-data layer (query_grouped) +``` Upstream ask ([#59](https://github.com/RaredonLab/TissuePlex/issues/59)): designate a sending @@ -126,11 +138,18 @@ want *opposite* things from sampling: rare subset draws that subset at full density rather than a tenth of it. That ordering is the whole point of resolving filters server-side (issue #45). -One query cannot do both, because density runs last (above). Returning every edge -with a `passes_filter` boolean would sample the rare subset away along with -everything else, quietly undoing #45 — the flag would be evaluated *after* the -sample had already thinned the rows it applies to. So: two requests, each keeping -filter-then-sample internally, with separate density controls. +One query cannot do both. Returning every edge with a `passes_filter` boolean +would sample the rare subset away along with everything else — the flag would be +read *after* the sample had thinned the rows it describes. So: two requests. + +**They must nonetheless select the same edges**, and this was the one real bug in +the first implementation. Two independent `USING SAMPLE` draws at 10% overlap only +about 1% of the time, so edge data appeared where the graph beneath it had been +sampled away. `density_predicate` replaces bernoulli with a deterministic hash of +the edge id: every edge gets the same verdict in every query, so the predicate +commutes with the filters — density-then-filter and filter-then-density are the +same set — and B ⊆ A holds at every density. It is also stable across re-fetches, +where bernoulli flickers on each pan. Sizing, measured across the bundled datasets — unique directed edges: @@ -143,11 +162,17 @@ Sizing, measured across the bundled datasets — unique directed edges: Rendering 169K–300K line segments is not the constraint; deck.gl handles that. The payload is — `query-grouped` returns ~15 fields per edge, so a whole-tissue -view of a real run is tens of MB. The tissue graph therefore keeps a density -slider of its own, and the existing 500K-row cap stays as the backstop. A -**lean projection** for this endpoint (`x1,y1,x2,y2` and nothing else) is worth -doing at the same time: the structural layer draws lines and needs no scores, -types or LRM counts, which is most of the payload. +view of a real run is tens of MB. The lean projection was therefore done up +front rather than deferred: `edge` plus the four coordinates, measured **2.2–2.4× +smaller** than the grouped payload (cosmx 17.6 MB vs 42.0 MB). The 500K-row cap +stays as the backstop. + +**A second density slider was built and then removed.** The argument for one was +that the graph draws far more lines than the filtered edge layer. It does not +hold: unfiltered the two draw exactly the same number, and the graph already has +an *opacity* control, which is the better lever for clutter — sampling down a +layer whose purpose is to show complete structure misrepresents it. One slider +now drives both. ### Two cell-metadata pickers, sending and receiving diff --git a/docs/index.html b/docs/index.html index c4d72e4..2c2ee78 100644 --- a/docs/index.html +++ b/docs/index.html @@ -904,10 +904,9 @@

What the filter affects

  • Cell segments — only matching cells are drawn, and only they can be clicked or captured by a region selection.
    • -
  • Edges and the tissue graph — an edge is drawn only when - both of its endpoints match. The point of narrowing to two or three - cell types is the signalling within that subset, and an edge with one endpoint - outside would run off to a cell that is not on screen.
    • +
  • Nothing else. The cell filter does not touch edges or the + tissue graph — filtering cells and filtering edges are independent actions. To + narrow the edges, use the sending and receiving filters described below.
  • Transcripts are not filtered. Several platforms ship no transcript-to-cell assignment at all, so there is nothing to match on.
@@ -921,15 +920,50 @@

What the filter affects

Cell Segments updates to describe the subset.
-

Filtering edges

-

The Edge Filter inside Edge - Data works the same way but on columns of the edge table itself, including - annotations from an edge-metadata/ folder — a curation call, a - confidence score, a review flag. It composes with the cell filter: set both and - you see only the annotated edges among your chosen cells.

-

This is distinct from the LRM Mechanisms checklist - below it. The checklist chooses which mechanisms are scored on every - edge; the edge filter chooses which cell pairs are drawn at all.

+

Filtering edges by sending and receiving cell

+

Edge Filter inside Edge Data + has two dropdowns side by side — Sending cell and + Receiving cell. Each offers any cell metadata + column, the same list the cell filter uses, and each constrains one end + of an edge.

+ +
    +
  • Set one and that end is constrained while the other is + free — "everything sent from crypt cells, to anywhere".
    • +
  • Set both and you get the intersection: senders in one + group, receivers in the other. This is how you ask for + fibroblast → endothelial signalling and nothing else.
    • +
+ +
+

These are independent of the Cell Filter above. Narrowing + the cells you draw does not narrow the edges, and vice versa. An edge may + therefore terminate on a cell that is not currently drawn — that is intended, + so the graph structure can be read on its own terms.

+
+ +

Below the two dropdowns, a third filter works on columns of the + edge table itself, including annotations from an + edge-metadata/ folder — a curation call, a confidence score, a + review flag.

+ +

All of these are distinct from the LRM Mechanisms + checklist. The checklist chooses which mechanisms are scored on every + edge; the filters choose which cell pairs are drawn at all.

+ +

The order things happen in

+

Understanding this makes the controls predictable:

+
    +
  1. every edge in view
    2. +
  2. density — keeps a spatially random fraction
    3. +
  3. the result is the Tissue Graph layer
    4. +
  4. sending, then receiving, then the + edge-table filter
    5. +
  5. the result is the Edge Data layer
    6. +
+

Because the edge data is filtered from the tissue graph, it can never + appear where there is no graph edge beneath it. Both layers respond to the one + density slider and stay aligned as you move it.

@@ -1002,6 +1036,16 @@

Tissue Graph

showing which cells are spatial neighbors, independent of any LRM information. Each line indicates spatial adjacency between two cells. Use the checkbox to toggle it and the opacity slider to adjust its visual weight.

+ +
+

The tissue graph is ground truth: the total set of edges in + view. No filter narrows it — it is shown or hidden, nothing else. Only the + density slider changes how many of its lines are drawn, and that applies + equally to the Edge Data layer on top, so the two always align.

+

Use opacity, not density, to make it recede behind the edge + data. Lowering density removes lines from a layer whose whole purpose is to + show the complete structure.

+
diff --git a/frontend/package.json b/frontend/package.json index 99c0f97..1d71b7a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "tissueplex", - "version": "0.8.5", + "version": "0.8.6", "private": true, "scripts": { "dev": "vite", diff --git a/frontend/src/components/LayerPanel.jsx b/frontend/src/components/LayerPanel.jsx index ca384f8..4a90166 100644 --- a/frontend/src/components/LayerPanel.jsx +++ b/frontend/src/components/LayerPanel.jsx @@ -136,7 +136,7 @@ export default function LayerPanel() {
Tissue Graph
- +
Edge Data
@@ -740,6 +740,70 @@ function CellFilterSection({ unitLabel = "cell" }) { ); } +/** + * Issue #59 — the two endpoint filters, side by side. + * + * Both select from *cell* metadata, the same vocabulary the cell filter offers, + * and each constrains one end of an edge. Set both and you get the intersection: + * senders in A, receivers in B. Leave one unset and that end is unconstrained. + * + * These resolve against the cells table rather than the edge file's own + * `sending_type` / `receiving_type`, which are absent on five of six platforms as + * the r/ export scripts stand, carry one label where any cell column is wanted, + * and are frozen at scoring time. See docs/edge_filter_independence.md. + * + * Independent of the Cell Filter above it: filtering cells and filtering edges + * are separate actions, so an edge may terminate on a cell that is not drawn. + */ +function EndpointFilterSection() { + const { apiBase, sendingFilter, setSendingFilter, + receivingFilter, setReceivingFilter, categoricalOverrides } = usePanelSettings(); + const datasets = useActiveDatasets(); + const columns = useUnionList((d) => + fetch(`${apiBase}/spatial/${d}/cells/schema`) + .then((r) => (r.ok ? r.json() : null)) + .then((x) => (x?.columns ? Object.keys(x.columns) : []))); + + const fetchValues = React.useCallback((field) => { + const categorical = categoricalOverrides[`cell::${field}`] ?? null; + return mergeColorValues(datasets.map((d) => + fetch(`${apiBase}/spatial/${d}/color-values`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ mode: "metadata", field, categorical }), + }).then((r) => (r.ok ? r.json() : null)).catch(() => null))); + }, [apiBase, datasets.join(" "), categoricalOverrides]); // eslint-disable-line + + if (!columns.length) return null; + + const side = (label, filter, setFilter) => ( + // minWidth 0 lets the select shrink inside the flex row instead of forcing + // the sidebar to scroll sideways. +
+
{label}
+ +
+ ); + + return ( +
+
+ {side("Sending cell", sendingFilter, setSendingFilter)} + {side("Receiving cell", receivingFilter, setReceivingFilter)} +
+ {sendingFilter && receivingFilter && ( +
+ showing edges from {sendingFilter.field} + {" → "}{receivingFilter.field} only +
+ )} +
+ ); +} + function EdgeFilterSection() { const { apiBase, edgeFilter, setEdgeFilter, categoricalOverrides } = usePanelSettings(); const active = useActivePanels(); @@ -1380,24 +1444,42 @@ const CHIP_STYLE = { }; // ── Density row (top-level — applies to tissue graph + edge data) ───────────── -function DensityRow() { - const { edgeDensity, setEdgeDensity } = usePanelSettings(); +/** + * A rendering-volume control, not a filter. + * + * One slider drives both the tissue graph and the edge-data layer. They are + * separate *requests* — the graph is never filtered — but they do not need + * separate volume controls: unfiltered the two draw the same number of lines, + * and when the edge layer is filtered down, the graph's own opacity (5% by + * default) is the better lever for clutter. Sampling the graph to reduce clutter + * would misrepresent the structure of something that is meant to be ground truth. + * + * Sampling is applied last on both paths, after every filter, so lowering it + * never changes *which* edges qualify — only how many of them are drawn. + */ +function DensityRow({ label, value, onChange, note }) { return (
- density: {Math.round(edgeDensity * 100)}%{edgeDensity >= 1.0 ? " (all)" : ""} - tissue graph + edges + {label}: {Math.round(value * 100)}%{value >= 1.0 ? " (all)" : ""} + {note}
setEdgeDensity(parseFloat(e.target.value))} + value={value} + onChange={(e) => onChange(parseFloat(e.target.value))} style={{ width: "100%", accentColor: "#888", cursor: "pointer" }} />
); } +function EdgeDensityRow() { + const { edgeDensity, setEdgeDensity } = usePanelSettings(); + return ; +} + // ── Tissue graph section ────────────────────────────────────────────────────── function TissueGraphSection() { const { layers, setLayerProp } = usePanelSettings(); @@ -1743,6 +1825,7 @@ function EdgeSection() { attribute of the pair (a curation call, a confidence), whereas the checklist subsets the mechanisms scored on every edge. */}
Edge Filter
+ {/* ── LRM Mechanisms checklist ─────────────────────────────── */} diff --git a/frontend/src/components/Viewer.jsx b/frontend/src/components/Viewer.jsx index 00f5f0f..f2ec790 100644 --- a/frontend/src/components/Viewer.jsx +++ b/frontend/src/components/Viewer.jsx @@ -136,7 +136,8 @@ function ViewerPanel({ panelIndex }) { autocrineRadius, autocrineLineWidth, hiddenLrms, cellColorClamp, edgeColorClamp, setEdgeColorClamp, linkColorScale, - categoricalOverrides, cellFilter, edgeFilter, + categoricalOverrides, cellFilter, + sendingFilter, receivingFilter, edgeFilters, annotationMode, clearZoomMatch, activeRegion, addRegionPoint, cancelActiveRegion, commitRegion, @@ -628,10 +629,18 @@ function ViewerPanel({ panelIndex }) { patch({ cellBoundaryStats: { shown: cellPolygons.length, total: cellBoundaryTotal } }); }, [cellPolygons.length, cellBoundaryTotal]); // eslint-disable-line - const { edges, loading: edgesLoading } = useEdges( - apiBase, dataset, viewport, imageSize, edgesVisible || tissueGraphVisible, + // `edges` is the filtered edge-data layer; `graphEdges` is the unfiltered + // tissue graph. Two arrays from two requests — the graph is ground truth and + // must never be narrowed by an edge filter. cellFilter is deliberately absent: + // filtering cells and filtering edges are independent actions. + const { edges, graphEdges, loading: edgesLoading } = useEdges( + apiBase, dataset, viewport, imageSize, edgesVisible, edgeMinStrength, hiddenLrms, lrmCatalogue, edgeDensity, edgeFile, - cellFilter, edgeFilter + { sendingFilter, receivingFilter, edgeFilters, + // One density drives both fetches. It is a rendering-volume control, and + // the graph has its own opacity for visual weight — a better lever for + // clutter than sampling, which would misrepresent the structure. + graphEnabled: tissueGraphVisible, graphDensity: edgeDensity } ); // Explicit categorical/continuous choice for the active color-by column, or @@ -859,7 +868,7 @@ function ViewerPanel({ panelIndex }) { const tissueGraphLayer = new LineLayer({ id: "tissue-graph", - data: allDirectedEdges, + data: graphEdges, modelMatrix: rotModelMatrix, visible: tissueGraphVisible, opacity: tissueGraphOpacity, diff --git a/frontend/src/hooks/useEdges.js b/frontend/src/hooks/useEdges.js index a1d0624..1a0f6f2 100644 --- a/frontend/src/hooks/useEdges.js +++ b/frontend/src/hooks/useEdges.js @@ -47,16 +47,31 @@ function filterBody(filter) { export function useEdges( apiBase, dataset, viewport, imageSize, enabled, minStrength, hiddenLrms, lrmCatalogue, density = 1.0, - edgeFile = "edges.parquet", cellFilter = null, edgeFilter = null + edgeFile = "edges.parquet", + { sendingFilter = null, receivingFilter = null, edgeFilters = null, + graphEnabled = false, graphDensity = 1.0 } = {} ) { // Which edge-source parquet to query; appended to every /edges request. const efParam = `?edge_file=${encodeURIComponent(edgeFile)}`; - // Serialised so the structural effect can depend on filter *content* rather - // than object identity, which changes on every render. - const cellFilterBody = filterBody(cellFilter); - const edgeFilterBody = filterBody(edgeFilter); - const filterKey = JSON.stringify([cellFilterBody ?? null, edgeFilterBody ?? null]); + // Serialised so the effects can depend on filter *content* rather than object + // identity, which changes on every render. + const sendingBody = filterBody(sendingFilter); + const receivingBody = filterBody(receivingFilter); + const edgeFilterBodies = (edgeFilters ?? []).map(filterBody).filter(Boolean); + const filterKey = JSON.stringify( + [sendingBody ?? null, receivingBody ?? null, edgeFilterBodies]); + + // ── Tissue-graph state ──────────────────────────────────────────────────── + // A separate fetch, and separate on purpose. The tissue graph is the total set + // of edges — ground truth, shown or hidden, never subset by a filter. It also + // cannot share the edge-data request: that one filters *then* samples so a rare + // subset draws at full density, while the graph must not filter at all. A + // `passes_filter` flag on one shared result would be read after the sample had + // already thinned the rows it describes. + const [graphEdges, setGraphEdges] = useState([]); + const graphTimerRef = useRef(null); + const graphAbortRef = useRef(null); // ── Structural state ────────────────────────────────────────────────────── const [structuralEdges, setStructuralEdges] = useState([]); const [loadingStructural, setLoadingStructural] = useState(false); @@ -92,11 +107,13 @@ export function useEdges( const { xmin, ymin, xmax, ymax } = viewport; const body = { xmin, ymin, xmax, ymax, density: Math.max(0.01, Math.min(1.0, density)) }; if (minStrength != null && minStrength > 0) body.min_strength = minStrength; - // Metadata filters go to the server so they apply before the density - // sample; filtering the response instead would sample first and leave a - // fraction of the subset. - if (cellFilterBody) body.cell_filter = cellFilterBody; - if (edgeFilterBody) body.edge_filter = edgeFilterBody; + // Filters go to the server so they apply before the density sample; + // filtering the response instead would sample first and leave a fraction + // of the subset. The cell layer's own filter is deliberately NOT sent — + // filtering cells and filtering edges are independent actions. + if (sendingBody) body.sending_filter = sendingBody; + if (receivingBody) body.receiving_filter = receivingBody; + if (edgeFilterBodies.length) body.edge_filters = edgeFilterBodies; try { const res = await fetch(`${apiBase}/edges/${dataset}/query-grouped${efParam}`, { @@ -116,6 +133,37 @@ export function useEdges( return () => clearTimeout(structTimerRef.current); }, [apiBase, dataset, viewport, imageSize, enabled, minStrength, density, efParam, filterKey]); // eslint-disable-line + // ── Effect 1b: tissue-graph fetch ───────────────────────────────────────── + // Takes no filter arguments at all, and depends on no filter state — so no + // future edit can narrow the structural graph by wiring one through. Its only + // inputs are the viewport and its own density. + useEffect(() => { + if (!graphEnabled || !viewport || !imageSize?.w) { + setGraphEdges([]); + return; + } + clearTimeout(graphTimerRef.current); + graphTimerRef.current = setTimeout(async () => { + if (graphAbortRef.current) graphAbortRef.current.abort(); + const ctrl = new AbortController(); + graphAbortRef.current = ctrl; + const { xmin, ymin, xmax, ymax } = viewport; + try { + const res = await fetch(`${apiBase}/edges/${dataset}/query-structure${efParam}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ xmin, ymin, xmax, ymax, + density: Math.max(0.01, Math.min(1.0, graphDensity)) }), + signal: ctrl.signal, + }); + setGraphEdges(res.ok ? await res.json() : []); + } catch (e) { + if (e.name !== "AbortError") setGraphEdges([]); + } + }, DEBOUNCE_MS); + return () => clearTimeout(graphTimerRef.current); + }, [apiBase, dataset, viewport, imageSize, graphEnabled, graphDensity, efParam]); // eslint-disable-line + // ── Effect 2: score fetch ────────────────────────────────────────────────── // Runs when viewport OR hiddenLrms changes. // Short-circuits when hiddenLrms is empty (no filter → use score_sum from structural). @@ -230,5 +278,7 @@ export function useEdges( }; }, []); - return { edges, loading: loadingStructural || loadingScores }; + // graphEdges is returned separately from edges — the tissue-graph layer must + // read the unfiltered set, never the filtered one. + return { edges, graphEdges, loading: loadingStructural || loadingScores }; } diff --git a/frontend/src/store.js b/frontend/src/store.js index 1f80058..08ab68d 100644 --- a/frontend/src/store.js +++ b/frontend/src/store.js @@ -56,7 +56,15 @@ export function makeSettings() { hiddenLrms: new Set(), selectedGenes: null, // null = no filter; Set = allowlist cellFilter: null, - edgeFilter: null, + // Edge-side filters (issue #59). Independent of cellFilter: filtering cells + // and filtering edges are separate actions, and an edge may terminate on a + // cell that is not drawn. + // sending/receivingFilter — a *cell* metadata predicate on one endpoint; + // both set gives the intersection, one set leaves the other end free. + // edgeFilters — predicates on the edge table / edge-metadata, and-ed. + sendingFilter: null, + receivingFilter: null, + edgeFilters: [], categoricalOverrides: {}, // "cell::" | "edge::" -> bool categoryColorOverrides: {}, // "::" -> [r,g,b,a] transcriptColorOverrides: {}, // gene -> [r,g,b,a] @@ -76,6 +84,7 @@ function cloneSettings(src) { selectedGenes: src.selectedGenes === null ? null : new Set(src.selectedGenes), layers: Object.fromEntries( Object.entries(src.layers).map(([k, v]) => [k, { ...v }])), + edgeFilters: [...(src.edgeFilters ?? [])], categoricalOverrides: { ...src.categoricalOverrides }, categoryColorOverrides: { ...src.categoryColorOverrides }, transcriptColorOverrides: { ...src.transcriptColorOverrides }, @@ -114,7 +123,13 @@ export function sanitiseSettings(next, allowed, sameDataset) { next.edgeColorBy = { mode: "lrm_set", field: null }; } if (next.cellFilter && !has(cellFields, next.cellFilter.field)) next.cellFilter = null; - if (next.edgeFilter && !has(edgeFields, next.edgeFilter.field)) next.edgeFilter = null; + // The endpoint filters name *cell* columns even though they filter edges, so + // they validate against cellFields — getting this wrong would drop every one of + // them against a target whose edge table simply has different columns. + if (next.sendingFilter && !has(cellFields, next.sendingFilter.field)) next.sendingFilter = null; + if (next.receivingFilter && !has(cellFields, next.receivingFilter.field)) next.receivingFilter = null; + // Every filter in the list is validated, not just the first. + next.edgeFilters = (next.edgeFilters ?? []).filter((f) => has(edgeFields, f.field)); if (next.selectedGenes && genes) { const keep = [...next.selectedGenes].filter((g) => genes.has(g)); @@ -211,7 +226,9 @@ export const useStore = create((set, get) => ({ hiddenLrms: new Set(), categoricalOverrides: {}, cellFilter: null, - edgeFilter: null, + sendingFilter: null, + receivingFilter: null, + edgeFilters: [], categoryColorOverrides: {}, transcriptColorOverrides: {}, colorBy: { mode: "off", field: null }, @@ -251,7 +268,7 @@ export const useStore = create((set, get) => ({ // Mechanism and edge-column names are specific to one edge file, so the // panel that changed is always cleared; the others only while linked. const panels = next.map((p, idx) => (idx === i || s.linkSettings) - ? { ...p, settings: { ...p.settings, hiddenLrms: new Set(), edgeFilter: null } } + ? { ...p, settings: { ...p.settings, hiddenLrms: new Set(), edgeFilters: [] } } : p); return { panels, selection: sel }; }), @@ -281,7 +298,8 @@ export const useStore = create((set, get) => ({ // to a rare cluster shows all of it rather than a sample of a sample. // // cellFilter also governs edges: an edge is drawn only when BOTH endpoints - // survive it. edgeFilter is independent and applies to the edge table itself. + // survive it — superseded by sendingFilter/receivingFilter, which apply to one + // endpoint each and are independent of the cell layer entirely (issue #59). // ── Shared colour scale across panels ───────────────────────────────────── // On by default, and this is a figure-integrity setting rather than a @@ -462,7 +480,16 @@ export const useStore = create((set, get) => ({ setAutocrineLineWidth: (v) => get().patchSettings({ autocrineLineWidth: v }), setCellFilter: (f) => get().patchSettings({ cellFilter: f }), - setEdgeFilter: (f) => get().patchSettings({ edgeFilter: f }), + setSendingFilter: (f) => get().patchSettings({ sendingFilter: f }), + setReceivingFilter: (f) => get().patchSettings({ receivingFilter: f }), + // Edge-table filters are a list, and-ed together. Index-addressed so the UI + // can add, replace and remove rows without rebuilding the array itself. + setEdgeFilterAt: (i, f) => { + const next = [...(get().getSetting("edgeFilters") ?? [])]; + if (f === null) next.splice(i, 1); else next[i] = f; + get().patchSettings({ edgeFilters: next.filter(Boolean) }); + }, + setEdgeFilters: (list) => get().patchSettings({ edgeFilters: list ?? [] }), // ── LRM mechanism filter ─────────────────────────────────────────────────── // Keyed on the "ligand|receptor" string, so a mechanism present in both diff --git a/frontend/src/store.settings.test.js b/frontend/src/store.settings.test.js index 0b8c9a0..3eaa63e 100644 --- a/frontend/src/store.settings.test.js +++ b/frontend/src/store.settings.test.js @@ -347,14 +347,27 @@ describe("2c — the guard drops what the target cannot honour", () => { expect(settings(1).colorBy).toEqual({ mode: "gene_set", field: null }); }); - it("drops an edge filter and edge colour-by naming missing columns", () => { - S().setEdgeFilter({ field: "nope", values: ["1"] }); + it("drops edge filters and edge colour-by naming missing columns", () => { + S().setEdgeFilters([{ field: "confidence", values: ["high"] }, + { field: "nope", values: ["1"] }]); S().setEdgeColorBy("metadata", "nope"); S().pushSettings(0, 1, allowed); - expect(settings(1).edgeFilter).toBeNull(); + // Every filter in the list is checked, not just the first. + expect(settings(1).edgeFilters).toEqual([{ field: "confidence", values: ["high"] }]); expect(settings(1).edgeColorBy).toEqual({ mode: "lrm_set", field: null }); }); + it("validates endpoint filters against CELL columns, not edge columns", () => { + // sendingFilter/receivingFilter name cell metadata even though they filter + // edges. Checking them against edgeFields would drop every one of them + // whenever the target's edge table happens to have different columns. + S().setSendingFilter({ field: "cluster", values: ["4"] }); + S().setReceivingFilter({ field: "absent_column", values: ["x"] }); + S().pushSettings(0, 1, allowed); + expect(settings(1).sendingFilter).toEqual({ field: "cluster", values: ["4"] }); + expect(settings(1).receivingFilter).toBeNull(); + }); + it("intersects the gene allowlist", () => { S().setSelectedGenes(new Set(["A", "ZZZ"])); S().pushSettings(0, 1, allowed); @@ -404,3 +417,53 @@ describe("2c — the guard drops what the target cannot honour", () => { expect(settings(1).cellColorClamp).toEqual({ low: 0, high: 4000 }); }); }); + +describe("#59 — edge filtering is independent of cell filtering", () => { + it("cellFilter and the endpoint filters are separate values", () => { + S().setCellFilter({ field: "region", values: ["crypt"] }); + expect(settings(0).sendingFilter).toBeNull(); + expect(settings(0).receivingFilter).toBeNull(); + + S().setSendingFilter({ field: "region", values: ["villus"] }); + // Setting one must not disturb the other, in either direction. + expect(settings(0).cellFilter).toEqual({ field: "region", values: ["crypt"] }); + }); + + it("edgeFilters is a list, and-ed", () => { + S().setEdgeFilters([{ field: "a", values: ["1"] }, { field: "b", values: ["2"] }]); + expect(settings(0).edgeFilters).toHaveLength(2); + S().setEdgeFilterAt(0, null); // remove the first + expect(settings(0).edgeFilters).toEqual([{ field: "b", values: ["2"] }]); + }); + + it("a dataset change clears every filter that names a column", () => { + S().setCellFilter({ field: "region", values: ["crypt"] }); + S().setSendingFilter({ field: "cluster", values: ["4"] }); + S().setReceivingFilter({ field: "region", values: ["mid"] }); + S().setEdgeFilters([{ field: "confidence", values: ["high"] }]); + + S().setPanelDataset(0, "other-dataset"); + expect(settings(0).cellFilter).toBeNull(); + expect(settings(0).sendingFilter).toBeNull(); + expect(settings(0).receivingFilter).toBeNull(); + expect(settings(0).edgeFilters).toEqual([]); + // ...but not the density, which names nothing. + expect(settings(0).edgeDensity).toBe(0.1); + }); + + it("an edge-file change clears edge-table filters but not endpoint filters", () => { + // Edge-table column names are specific to one file; cell metadata is not. + S().setSendingFilter({ field: "cluster", values: ["4"] }); + S().setEdgeFilters([{ field: "confidence", values: ["high"] }]); + S().setPanelEdgeFile(0, "edges/other.parquet"); + expect(settings(0).edgeFilters).toEqual([]); + expect(settings(0).sendingFilter).toEqual({ field: "cluster", values: ["4"] }); + }); + + it("cloneSettings does not alias the filter list across panels", () => { + S().setEdgeFilters([{ field: "a", values: ["1"] }]); + S().setActivePanel(0); + S().setLinkSettings(true); + expect(settings(0).edgeFilters).not.toBe(settings(1).edgeFilters); + }); +}); From 5e934925fd1cbf51cce53c4a1d14045b3e62e415 Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Tue, 18 Aug 2026 08:54:50 -0400 Subject: [PATCH 11/15] chore: v0.8.6 Co-Authored-By: Claude Opus 5 From 9137c973ada29e76c4a647f3800f1cb98331283b Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:42:13 -0400 Subject: [PATCH 12/15] feat: local neighbourhood highlighting and summaries (#60) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Click a cell, press "show local neighbourhood", and see what it is connected to in the tissue graph — drawn on the canvas and summarised in the info panel: neighbour and edge counts, the enclosing radius in µm, composition by any cell metadata column, and the strongest LRMs across its incident edges. The design note is docs/neighborhood_summary.md. Computed server-side, and that is the point rather than an implementation detail. The obvious version filters the `edges` array the frontend already holds; it would appear to work and be wrong twice over, since that array is density-sampled (nine of ten neighbours missing at the default) and viewport-bounded (a neighbour just off-screen does not exist, and the answer changes as you pan). A neighbourhood is a property of the tissue, not of the current view, so the query ignores density, the viewport, the endpoint filters and the LRM checklist. Cost was measured first and is not a reason to avoid it: 15 ms on the 3.8M-row CosMx file, no index, no cache. Two marks are drawn because the radius alone would mislead. Connectivity is anisotropic — a cell at a tissue boundary has neighbours on one side only — so a disc around it encloses many cells it is not connected to. The highlighted cells are the honest answer; the circle is the spatial scale the issue asked for. Composition resolves against the cells table rather than the edge file's sending_type, for the reasons recorded in v0.8.6: that column is absent on five of six platforms, carries one label, and is frozen at scoring time. It keys on the frame's cell_id *column* — the frame carries a plain RangeIndex, and my first version indexed by position, which matched nothing and reported every neighbour as missing. Also fixed, found while testing this: CellInfoPanel rendered stale `detail` for one frame after the selection cleared, because `detail` is state and outlives `selectedCell`. The new section dereferenced it and took the whole app to the error boundary on a dataset change. Guarded on both. Verified: - Counts and radius match ground truth computed independently from the parquet, on all 9 bundled datasets across 6 platforms; 3–125 ms each. - End to end through the real button on Xenium: 11 neighbours, 18 edges, 12.8 µm, the highlight landing exactly on the tissue graph vertices joined to the clicked cell — and visibly enclosing unconnected cells the circle would have claimed. - 56 frontend tests, including that the highlight carries its panel and is dropped on selection change or a dataset change in its own panel. - Golden snapshot 249 probes, edge pipeline check, DuckDB config check. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 34 +++++- backend/app/main.py | 2 +- backend/app/readers/edge_reader.py | 89 +++++++++++++++ backend/app/routers/edges.py | 42 +++++++ docs/index.html | 31 ++++++ docs/neighborhood_summary.md | 109 ++++++++++++++++++ frontend/package.json | 2 +- frontend/src/components/CellInfoPanel.jsx | 130 +++++++++++++++++++++- frontend/src/components/Viewer.jsx | 57 ++++++++++ frontend/src/store.annotations.test.js | 39 +++++++ frontend/src/store.js | 26 ++++- 11 files changed, 553 insertions(+), 8 deletions(-) create mode 100644 docs/neighborhood_summary.md diff --git a/CLAUDE.md b/CLAUDE.md index 89a0501..3993fe1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -109,7 +109,8 @@ backend/ tiles.py DZI descriptor + tile serving; auto-builds pyramid on first request spatial.py Platform-agnostic router: all /spatial/... endpoints xenium.py DEPRECATED — kept for reference; not registered in main.py - edges.py edge query, LRM catalogue, edge color values, edge detail; + edges.py edge query, LRM catalogue, edge color values, edge detail, + per-cell neighbourhood summary (#60); all endpoints take an edge_file param (multi-file support); /files lists edge sources (top-level + edges/ folder) layers.py generic parquet layer router @@ -347,6 +348,37 @@ source, and a supplemental column silently shadowing a real one would be painful --- +## Local Neighbourhood (issue #60) + +`GET /edges/{dataset}/neighborhood/{cell_id}?field=` returns every +cell the clicked one is joined to by any edge, plus counts, the enclosing radius +in µm, a breakdown by any cell metadata column, and the top LRMs over its +incident edges. + +**It is deliberately unfiltered and unsampled.** A neighbourhood is a property of +the tissue, not of the current view, so density, the viewport, the endpoint +filters and the LRM checklist are all ignored. This is also why it cannot be +computed in the frontend from the `edges` array already in memory: that array is +density-sampled (a tenth of the neighbours by default) and viewport-bounded, so +the answer would be silently short and would change as you pan. Cost is not a +reason to avoid the query — 15 ms on the 3.8M-row CosMx file. + +Composition resolves against the **cells table**, keyed on its `cell_id` column +(the frame carries a plain RangeIndex, so indexing by position matches nothing +and reports every neighbour as missing). Not the edge file's `sending_type` — +see the caveat under the edges.parquet schema. + +Two marks are drawn, and both are needed. The **connected cells** are the honest +answer, since connectivity is anisotropic: a cell at a tissue boundary has +neighbours on one side only, and the enclosing **circle** contains many cells it +is not connected to. The circle is what the issue asked for and gives the spatial +scale; the points say which cells actually count. + +`neighborhood` in the store carries its `panelIndex` like annotations and +selection, and is dropped whenever the selection changes or its own panel changes +dataset — a highlight left over from a previous cell would sit on unrelated +tissue and read as the answer for the cell now selected. + ## Multiple Edge Files (edges/ folder) A dataset can carry more than one edge set so users can flip between different diff --git a/backend/app/main.py b/backend/app/main.py index 776a1f3..5f55466 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -3,7 +3,7 @@ from app.routers import tiles, spatial, edges, layers -APP_VERSION = "0.8.6" +APP_VERSION = "0.8.7" app = FastAPI(title="TissuePlex API", version=APP_VERSION) diff --git a/backend/app/readers/edge_reader.py b/backend/app/readers/edge_reader.py index 2ee810e..4ede10f 100644 --- a/backend/app/readers/edge_reader.py +++ b/backend/app/readers/edge_reader.py @@ -357,6 +357,95 @@ def endpoint_filter_sql( conds.append(f'CAST("{col}" AS VARCHAR) {pred}') return (" AND ".join(conds), []) if conds else ("", []) + def neighborhood(self, cell_id: str, top_lrms: int = 12) -> dict | None: + """Everything a cell is connected to, unfiltered and unsampled (issue #60). + + The neighbourhood is a property of the tissue, not of the current view, so + this deliberately ignores density, the viewport and every filter. Computing + it from the frontend's `edges` array instead would look like it worked and + be wrong twice over: that array is density-sampled (a tenth of the + neighbours by default) and viewport-bounded (missing any neighbour just + off-screen, and changing as you pan). + + Direction is ignored for membership — a cell that only *sends* to this one + is still a neighbour — but the returned edge counts are directed, because + that is what the edge layer draws. + + Returns None when the cell appears in no edge at all, which is an ordinary + outcome rather than an error. + """ + cols = set(self._parquet_schema().names) + if not {"sending_cell", "receiving_cell"} <= cols: + return None + ps = self.pixel_size + has_lrm, has_score = "lrm" in cols, "score" in cols + + with self._conn() as conn: + rows = conn.execute(f""" + SELECT * FROM {self._from()} + WHERE CAST("sending_cell" AS VARCHAR) = ? + OR CAST("receiving_cell" AS VARCHAR) = ? + """, [str(cell_id), str(cell_id)]).df() + if rows.empty: + return None + + me = str(cell_id) + send = rows["sending_cell"].astype(str) + recv = rows["receiving_cell"].astype(str) + + # The partner cell, and the coordinates of whichever end is not us. Both + # come from the edge file, which is the one source every platform has — + # MERSCOPE and CosMx ship no boundary polygons to derive a centroid from. + partner = recv.where(send == me, send) + px = rows["x2"].where(send == me, rows["x1"]) if "x2" in rows else None + py = rows["y2"].where(send == me, rows["y1"]) if "y2" in rows else None + + self_x = rows["x1"].where(send == me, rows["x2"]) if "x1" in rows else None + self_y = rows["y1"].where(send == me, rows["y2"]) if "y1" in rows else None + centre = ([float(self_x.iloc[0]) / ps, float(self_y.iloc[0]) / ps] + if self_x is not None and len(self_x) else None) + + # One point per distinct partner. Autocrine rows name us as the partner; + # they are counted separately rather than drawn as a neighbour of ourself. + pts, radius_px = [], 0.0 + if px is not None: + frame = {} + for pid, x, y in zip(partner, px, py): + if pid == me or pid in frame: + continue + frame[pid] = (float(x) / ps, float(y) / ps) + pts = [{"cell_id": k, "x": v[0], "y": v[1]} for k, v in frame.items()] + if centre and pts: + radius_px = max(((q["x"] - centre[0]) ** 2 + + (q["y"] - centre[1]) ** 2) ** 0.5 for q in pts) + + neighbours = sorted({p for p in partner if p != me}) + + lrms = [] + if has_lrm: + agg = rows.groupby("lrm", dropna=True) + summed = (agg["score"].sum() if has_score else agg.size()).sort_values(ascending=False) + lrms = [{"lrm": str(k), + "value": float(v) if has_score else int(v), + "n": int(agg.size()[k])} + for k, v in summed.head(top_lrms).items()] + + return { + "cell_id": me, + "center": centre, + "neighbors": neighbours, + "neighbor_points": pts, + "n_neighbors": len(neighbours), + # Directed edges incident to this cell, deduplicated by edge id — the + # raw rows are one per (edge x LRM), which would overcount by ~500x. + "n_edges": int(rows["edge"].nunique()) if "edge" in rows else len(rows), + "n_autocrine": int(((send == me) & (recv == me)).any()), + "radius_px": radius_px, + "radius_um": radius_px * ps, + "lrm_composition": lrms, + "score_basis": "score" if has_score else "count", + } + def edge_detail(self, edge_id: str) -> dict | None: """Return all LRM rows for a single directed edge, structured for the info panel.""" with self._conn() as conn: diff --git a/backend/app/routers/edges.py b/backend/app/routers/edges.py index c2d7f76..36912f8 100644 --- a/backend/app/routers/edges.py +++ b/backend/app/routers/edges.py @@ -329,6 +329,48 @@ def edge_color_values(dataset: str, body: EdgeColorRequest, ) +@router.get("/{dataset}/neighborhood/{cell_id:path}") +def cell_neighborhood(dataset: str, cell_id: str, + field: Optional[str] = Query(None), + edge_file: str = Query("edges.parquet")): + """Everything one cell is connected to, plus a summary of it (issue #60). + + Unfiltered and unsampled by design: a neighbourhood describes the tissue, not + the current view, so density, the viewport and every filter are ignored. + + `field` names a *cell* metadata column to break the neighbourhood down by. + Composition comes from the cells table rather than the edge file's + sending_type — that column is absent on five of six platforms, carries one + label, and is frozen at scoring time (see docs/edge_filter_independence.md). + """ + nb = _reader(dataset, edge_file).neighborhood(cell_id) + if nb is None: + # Not an error: a cell with no edges is an ordinary outcome, and the panel + # says "no connections" rather than showing an empty summary. + return {"cell_id": cell_id, "n_neighbors": 0, "n_edges": 0, + "neighbors": [], "neighbor_points": [], "lrm_composition": []} + + if field and nb["neighbors"]: + from app.routers import spatial + try: + frame = spatial._reader(dataset)._metadata_frame() + except Exception: + frame = None + # Keyed on the cell_id *column*, matching filter_cell_ids — the frame + # carries a plain RangeIndex, so indexing by position would silently + # match nothing and report every neighbour as missing. + if frame is not None and field in frame.columns and "cell_id" in frame.columns: + ids = frame["cell_id"].astype(str) + sub = frame.loc[ids.isin(set(nb["neighbors"])), field].dropna() + counts = sub.astype(str).value_counts() + nb["composition_field"] = field + nb["composition"] = [{"value": k, "n": int(v)} for k, v in counts.items()] + # Neighbours the column says nothing about are reported rather than + # dropped, so the parts always add up to n_neighbors. + nb["composition_missing"] = nb["n_neighbors"] - int(counts.sum()) + return nb + + @router.get("/{dataset}/edge/{edge_id:path}") def edge_detail(dataset: str, edge_id: str, edge_file: str = Query("edges.parquet")): diff --git a/docs/index.html b/docs/index.html index 2c2ee78..6e2e23b 100644 --- a/docs/index.html +++ b/docs/index.html @@ -1170,6 +1170,37 @@

Clicking on a Cell

shows the barcode ID, all metadata values for that cell, and the current color-by value highlighted. Click on empty canvas to dismiss.

+

The Local Neighbourhood

+

With a cell selected, show local neighbourhood at the + bottom of the info panel reveals everything that cell is connected to in the + tissue graph. The canvas marks the clicked cell in white, its connected + neighbours in yellow, and draws a circle at the distance of the furthest one. + The panel reports:

+ +
    +
  • neighbours and local edges — how many + cells it touches, and how many directed edges that is;
    • +
  • radius in µm, so the reach can be quoted without + measuring it off the screen;
    • +
  • composition — the neighbours broken down by whichever + metadata column you are colouring cells by;
    • +
  • mechanisms — the strongest LRMs across its edges.
    • +
+ +
+

The circle is a convenience, not the answer. Connectivity is not evenly + spread — a cell at the edge of a tissue has neighbours on one side only — so + the circle usually contains cells it is not connected to. The yellow + marks are the neighbourhood; the circle only shows how far it reaches.

+
+ +
+

The summary is computed on the full data, so it is unaffected by the + density slider, by the sending and receiving filters, or by how far you are + zoomed in. It describes the tissue, not the current view. Clicking another + cell replaces it; hide highlight removes it.

+
+

Clicking on an Edge or Tissue Graph Line

Click on any line in the Tissue Graph or Directed Edges layer to open the Edge Info Panel, showing the sending and receiving cell diff --git a/docs/neighborhood_summary.md b/docs/neighborhood_summary.md new file mode 100644 index 0000000..4172f4a --- /dev/null +++ b/docs/neighborhood_summary.md @@ -0,0 +1,109 @@ +# Issue #60 — local neighbourhood highlighting and summaries + +Status: **implemented** in v0.8.7. Kept as the design record. + +`GET /edges/{dataset}/neighborhood/{cell_id}?field=` — unfiltered +and unsampled. The panel shows a **show local neighbourhood** button under the +cell detail; the canvas draws the connected cells, the clicked cell, and the +enclosing radius. + +Upstream ask +([#60](https://github.com/RaredonLab/TissuePlex/issues/60)): click a cell, press a +button, and (a) see the local neighbourhood — the cells it is connected to in the +tissue graph — drawn on the canvas, and (b) get summary metrics: composition by +cell type or selected metadata, number of neighbours, number of local edges, +local edge composition. + +--- + +## The trap: this cannot be computed client-side + +The obvious implementation is to filter the `edges` array the frontend already +holds. It would appear to work and be **silently wrong**, for two independent +reasons: + +- that array is **density-sampled** — 10% by default, so nine of ten neighbours + are missing; +- it is **viewport-bounded**, so a neighbour just off-screen does not exist as + far as the client is concerned, and the answer changes when you pan. + +A neighbourhood is a property of the tissue, not of the current view. It needs a +backend query, and that query must ignore both density and the viewport. + +Cost is not a concern — measured on the incident-edge scan: + +| dataset | rows in file | incident rows | scan | +|---|---|---|---| +| cosmx-mousebrain | 3,819,698 | 172 | **15 ms** | +| xenium_human_breast_2fov | 134,277 | 30 | 3 ms | +| mouse_ileum_tiny | 669 | 30 | 1 ms | + +No index, no cache. + +## Definition + +**The neighbourhood is every cell joined to the clicked cell by any edge in the +file** — unfiltered and unsampled, consistent with the tissue graph being ground +truth (issue #59). Direction is ignored for membership: a cell that only *sends* +to the clicked cell is still a neighbour. + +Deliberately *not* affected by: density, the viewport, the sending/receiving +filters, the edge-table filters, the cell filter, or the LRM checklist. Those +control what is drawn; this describes what the tissue is. + +## Drawing it + +The issue asks for "a local radius that encompasses all of the cells within that +cell's neighborhood". Two things get drawn, because the radius alone would +mislead: + +- **the connected cells themselves**, highlighted — this is the honest answer, since + connectivity is anisotropic. A cell at a tissue boundary has neighbours on one + side only, and a disc around it encloses many cells it is not connected to. +- **a circle at the enclosing radius**, which is what was asked for and gives the + spatial scale at a glance. + +The radius is also reported numerically, in µm via the panel's `pixel_size`, so +it can be quoted without measuring off the screen. + +## Summary metrics + +| metric | source | +|---|---| +| neighbours | distinct partner cells | +| local edges | directed edges incident to the cell | +| autocrine | whether the cell signals to itself | +| radius | max centroid distance, µm | +| composition | any cell metadata column, counted over the neighbours | +| edge composition | top LRMs over the incident edges, by score | + +Composition uses the **cells table**, not the edge file's `sending_type` — same +reasoning as #59: that column is absent on five of six platforms, carries one +label, and is frozen at scoring time. Any cell metadata column is offerable, and +the panel defaults to the active colour-by field when there is one, since that is +what the user is already looking at. + +## Surface + +| | | +|---|---| +| `backend/app/readers/edge_reader.py` | `neighborhood(cell_id)` — incident edges, partners, radius, LRM composition | +| `backend/app/routers/edges.py` | `GET /{dataset}/neighborhood/{cell_id}`; joins cell metadata for composition | +| `frontend/src/components/CellInfoPanel.jsx` | a **Neighbourhood** button and the summary block | +| `frontend/src/components/Viewer.jsx` | two layers — highlighted neighbours, and the radius circle | +| `frontend/src/store.js` | `neighborhood` selection state, cleared with the cell selection | + +The `/edge/{edge_id}` detail endpoint is the precedent for shape and for how the +info panel consumes it. + +## Consequences to get right + +- **Per panel.** The highlight belongs to the panel that was clicked, like + annotations and selection. `selection` already carries `panelIndex`. +- **Clearing.** Selecting another cell, changing dataset, or closing the panel must + drop the highlight, or it strands over unrelated tissue. +- **Platforms without boundaries.** MERSCOPE and CosMx return no polygons, so the + highlight cannot be a filled cell outline everywhere. Draw it as points at the + edge-file centroids, which every platform has. +- **A cell with no edges** is a normal outcome, not an error — the panel should say + "no connections" rather than render an empty summary. diff --git a/frontend/package.json b/frontend/package.json index 1d71b7a..8a1fe2e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "tissueplex", - "version": "0.8.6", + "version": "0.8.7", "private": true, "scripts": { "dev": "vite", diff --git a/frontend/src/components/CellInfoPanel.jsx b/frontend/src/components/CellInfoPanel.jsx index 1684d9d..289e864 100644 --- a/frontend/src/components/CellInfoPanel.jsx +++ b/frontend/src/components/CellInfoPanel.jsx @@ -68,7 +68,11 @@ export default function CellInfoPanel() { {loading &&

Loading…
} - {detail && !loading && ( + {/* `detail` is state and outlives `selectedCell` by one render when the + selection is cleared — changing dataset, for instance. Guarding on both + stops stale detail being shown for a cell that is no longer selected, + and stops the section below dereferencing a null selection. */} + {detail && selectedCell && !loading && ( <> {/* Color-by highlight — shown whenever cell coloring is active */} {colorByInfo && ( @@ -118,12 +122,136 @@ export default function CellInfoPanel() { ))} )} + + )} ); } +/** + * Issue #60 — what this cell is connected to in the tissue graph. + * + * Fetched on demand rather than with the cell detail: it is a second query, and + * most clicks are just "what is this cell". The result comes from the server + * unfiltered and unsampled — see the store's `neighborhood` note for why it + * cannot be derived from the edges the frontend already holds. + */ +function NeighborhoodSection({ dataset, cellId, panelIndex, unitLabel }) { + const apiBase = useStore((s) => s.apiBase); + const neighborhood = useStore((s) => s.neighborhood); + const setNeighborhood = useStore((s) => s.setNeighborhood); + const clearNeighborhood = useStore((s) => s.clearNeighborhood); + const panel = useStore((s) => s.panels[panelIndex]); + const { colorBy } = usePanelSettings(panelIndex); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const shown = neighborhood?.cellId === cellId ? neighborhood.data : null; + + const load = async () => { + setLoading(true); setError(null); + // Break the neighbourhood down by whatever the user is already colouring by; + // asking them to pick a column again would repeat a choice they just made. + const field = colorBy?.mode === "metadata" && colorBy.field ? colorBy.field : null; + const ef = `?edge_file=${encodeURIComponent(panel?.edgeFile ?? "edges.parquet")}`; + const q = field ? `${ef}&field=${encodeURIComponent(field)}` : ef; + try { + const r = await fetch( + `${apiBase}/edges/${dataset}/neighborhood/${encodeURIComponent(cellId)}${q}`); + if (!r.ok) { setError(`HTTP ${r.status}`); return; } + setNeighborhood({ panelIndex, cellId, data: await r.json() }); + } catch (e) { + setError(e.message); + } finally { + setLoading(false); + } + }; + + const pixelSize = panel?.pixelSize ?? 1; + + return ( + <> + + {!shown && ( + + )} + {error &&
{error}
} + + {shown && shown.n_neighbors === 0 && ( +
+ This {unitLabel} has no connections in the tissue graph. +
+ )} + + {shown && shown.n_neighbors > 0 && ( + <> + + + + {shown.n_autocrine > 0 && } + + {shown.composition?.length > 0 && ( + <> + + {shown.composition.map((c) => ( + + ))} + {shown.composition_missing > 0 && ( + + )} + + )} + {!shown.composition && ( +
+ Colour cells by a metadata column to see composition. +
+ )} + + {shown.lrm_composition?.length > 0 && ( + <> + + {shown.lrm_composition.map((m) => ( + + ))} + + )} + + + + )} + + ); +} + /** * Given a loaded cell detail and the current color-by state, return * { label, display } for the highlighted box, or null if nothing to show. diff --git a/frontend/src/components/Viewer.jsx b/frontend/src/components/Viewer.jsx index f2ec790..935f06e 100644 --- a/frontend/src/components/Viewer.jsx +++ b/frontend/src/components/Viewer.jsx @@ -178,6 +178,12 @@ function ViewerPanel({ panelIndex }) { // Annotations belong to the panel that drew them. Subscribe to the raw arrays // so a change re-renders, then narrow — the selectors on the store are plain // functions and would not themselves trigger an update. + // Issue #60. Scoped to this panel: the coordinates are this dataset's image + // pixels, so a highlight from the other panel would land nowhere meaningful. + const neighborhoodState = useStore((s) => s.neighborhood); + const neighborhood = neighborhoodState?.panelIndex === panelIndex + ? neighborhoodState.data : null; + const allRegions = useStore((s) => s.regions); const allMeasurements = useStore((s) => s.measurements); const activeRegionPanel = useStore((s) => s.activeRegionPanel); @@ -945,6 +951,54 @@ function ViewerPanel({ panelIndex }) { }); // ── Annotation layers ───────────────────────────────────────────────────── + // Two marks, because the radius alone would mislead. Connectivity is + // anisotropic — a cell at a tissue boundary has neighbours on one side only — + // so a disc around it encloses many cells it is not connected to. The points + // are the honest answer; the circle is the spatial scale that was asked for. + const neighborRingLayer = new ScatterplotLayer({ + id: "neighborhood-radius", + data: neighborhood?.center && neighborhood.radius_px > 0 ? [neighborhood] : [], + modelMatrix: rotModelMatrix, + getPosition: (d) => d.center, + getRadius: (d) => d.radius_px, + radiusUnits: "common", + filled: false, + stroked: true, + getLineColor: [255, 210, 80, 150], + getLineWidth: 1.5, + lineWidthMinPixels: 1, + pickable: false, + }); + const neighborPointLayer = new ScatterplotLayer({ + id: "neighborhood-cells", + data: neighborhood?.neighbor_points ?? [], + modelMatrix: rotModelMatrix, + getPosition: (d) => [d.x, d.y], + getRadius: 5, + radiusMinPixels: 3, + radiusMaxPixels: 9, + getFillColor: [255, 210, 80, 230], + stroked: true, + getLineColor: [40, 30, 0, 255], + lineWidthMinPixels: 0.5, + pickable: false, + }); + // The clicked cell itself, so the centre of the neighbourhood is unambiguous. + const neighborCenterLayer = new ScatterplotLayer({ + id: "neighborhood-center", + data: neighborhood?.center ? [neighborhood] : [], + modelMatrix: rotModelMatrix, + getPosition: (d) => d.center, + getRadius: 7, + radiusMinPixels: 4, + radiusMaxPixels: 12, + getFillColor: [255, 255, 255, 255], + stroked: true, + getLineColor: [255, 160, 0, 255], + lineWidthMinPixels: 1.5, + pickable: false, + }); + const regionFillLayers = regions.map((r) => new SolidPolygonLayer({ id: `region-fill-${r.id}`, @@ -1032,6 +1086,9 @@ function ViewerPanel({ panelIndex }) { const deckLayers = [ cellFillLayer, cellOutlineLayer, transcriptLayer, tissueGraphLayer, edgeDirectedLayer, edgeArrowheadLayer, edgeAutocrineLayer, + // Above the edges so the highlight reads against them, below the annotations + // so a region outline is never hidden by it. + neighborRingLayer, neighborPointLayer, neighborCenterLayer, ...regionFillLayers, ...regionOutlineLayers, activeRegionLayer, activeVertexLayer, measureLineLayer, measureEndpointLayer, measureFirstLayer, diff --git a/frontend/src/store.annotations.test.js b/frontend/src/store.annotations.test.js index 436f7f8..2b9d6d6 100644 --- a/frontend/src/store.annotations.test.js +++ b/frontend/src/store.annotations.test.js @@ -117,3 +117,42 @@ describe("clearing annotations is scoped to a panel", () => { expect(S().measurements).toEqual([]); }); }); + +describe("#60 — the neighbourhood highlight is scoped and cleared", () => { + const nb = { panelIndex: 1, cellId: "c1", data: { n_neighbors: 3 } }; + + beforeEach(() => { + useStore.setState({ neighborhood: null, selection: null }); + }); + + it("is dropped when another cell is selected", () => { + // A highlight left over from a previous cell would sit on unrelated tissue + // and read as the answer for the cell now selected. + useStore.setState({ neighborhood: nb }); + S().setSelectedCell({ cell_id: "c2" }, 1); + expect(S().neighborhood).toBeNull(); + }); + + it("is dropped when an edge is selected, and when selection is cleared", () => { + useStore.setState({ neighborhood: nb }); + S().setSelectedEdge({ edge: "a|b" }, 0); + expect(S().neighborhood).toBeNull(); + + useStore.setState({ neighborhood: nb }); + S().clearSelection(); + expect(S().neighborhood).toBeNull(); + }); + + it("is dropped when its own panel changes dataset", () => { + // The ids and coordinates belong to the dataset that was showing. + useStore.setState({ neighborhood: nb }); + S().setPanelDataset(1, "other-dataset"); + expect(S().neighborhood).toBeNull(); + }); + + it("survives a dataset change in the other panel", () => { + useStore.setState({ neighborhood: nb }); + S().setPanelDataset(0, "other-dataset"); + expect(S().neighborhood).toEqual(nb); + }); +}); diff --git a/frontend/src/store.js b/frontend/src/store.js index 08ab68d..1dfbd55 100644 --- a/frontend/src/store.js +++ b/frontend/src/store.js @@ -252,7 +252,8 @@ export const useStore = create((set, get) => ({ }); // Selections belong to a dataset; drop any that pointed at the old one. const sel = s.selection && s.selection.panelIndex === i ? null : s.selection; - return { panels, selection: sel }; + const nb = s.neighborhood && s.neighborhood.panelIndex === i ? null : s.neighborhood; + return { panels, selection: sel, neighborhood: nb }; }), // The LRM catalogue, colour range and edge filter are all specific to one @@ -279,11 +280,28 @@ export const useStore = create((set, get) => ({ // dataset and show the wrong cell. // { panelIndex, kind: "cell" | "edge", cell? , edge? } selection: null, + // Every selection change drops the neighbourhood with it. A highlight left + // over from a previous cell would sit on unrelated tissue and look like the + // answer for the cell now selected. setSelectedCell: (cell, panelIndex = 0) => - set({ selection: cell ? { panelIndex, kind: "cell", cell } : null }), + set({ selection: cell ? { panelIndex, kind: "cell", cell } : null, + neighborhood: null }), setSelectedEdge: (edge, panelIndex = 0) => - set({ selection: edge ? { panelIndex, kind: "edge", edge } : null }), - clearSelection: () => set({ selection: null }), + set({ selection: edge ? { panelIndex, kind: "edge", edge } : null, + neighborhood: null }), + clearSelection: () => set({ selection: null, neighborhood: null }), + + // ── Local neighbourhood (issue #60) ─────────────────────────────────────── + // { panelIndex, cellId, data } — data is the /neighborhood response. Carries + // its panel because the highlight is drawn in one panel only, like + // annotations and selection: the coordinates are that dataset's image pixels. + // + // Computed server-side and never from the frontend's `edges` array, which is + // density-sampled and viewport-bounded — deriving it here would silently + // under-count neighbours and change as you pan. + neighborhood: null, + setNeighborhood: (nb) => set({ neighborhood: nb }), + clearNeighborhood: () => set({ neighborhood: null }), // ── Categorical / continuous override (issue #35) ───────────────────────── // Keyed "cell::" / "edge::" → true | false. Absent means From 60599f8c6feef13576934288d7d93b74a71a7671 Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:42:13 -0400 Subject: [PATCH 13/15] chore: v0.8.7 Co-Authored-By: Claude Opus 5 From cb0f86b8174ce03acdc9f1ae54898bc6083c3e4c Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Tue, 18 Aug 2026 12:03:01 -0400 Subject: [PATCH 14/15] fix: report empty metadata columns, stop the pan re-render storm, version in README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three tidy-ups, plus a real hole found in my own test while doing them. Empty metadata columns. `fov` and `transcript_count` are entirely null on the bundled MERSCOPE dataset, but color-values reported them as a continuous 0–0 range. The UI then offered a range slider that did nothing, a legend with no span, and a filter that correctly matched no cells while looking broken. They now come back with `empty: True` and the filter section says "no values in this column". `type` is still set so nothing switching on categorical-vs-continuous needs a third case, and the cross-panel merge calls a column empty only when it is empty in every panel. The pan re-render storm. `usePanelSettings` subscribed to the whole store, so every sidebar section re-rendered on each OpenSeadragon viewport-change event. It now ignores `viewports` and `viewportActual`, which are the only continuously rewritten keys and which no consumer of that hook reads — ViewerPanel takes `viewports[panelIndex]` through its own selector and Match zoom reads `viewportActual` via getState(). Measured over one simulated pan of 120 writes: 121 re-renders before, 0 after, with ordinary settings changes still delivered. Narrowing the rest means a selector per section and is not worth it without component tests. Version in the README, and the bump list recorded in CLAUDE.md: three files move together — package.json, main.py, README — while every other v0.x.y in the tree is a historical reference that must not be swept along. Audited: the two authoritative files agreed, and all other mentions were correctly historical. The hole. tests/edge_pipeline_check.py called `color_values("metadata", None, field)`, but the signature is `(mode, field, genes, categorical)` — so the field went in as `genes`, the call returned the empty result, `spec_for` returned None, and the filtered-subset assertion was silently skipped on every dataset. The check printed OK while not testing its own core property. Fixed, and it now says so out loud when no filter resolves rather than passing over it. With the assertion actually running the property still holds, on all 9 datasets and 3 densities — so the earlier hand-verification was right, but the automated claim was weaker than stated. Golden snapshot 249 probes, edge pipeline check, DuckDB config check, 56 frontend tests. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 26 +++++++++++++++ README.md | 2 ++ backend/app/main.py | 2 +- backend/app/readers/base_reader.py | 17 ++++++++-- backend/tests/edge_pipeline_check.py | 12 +++++-- frontend/package.json | 2 +- frontend/src/components/LayerPanel.jsx | 22 ++++++++++--- frontend/src/hooks/usePanelSettings.js | 44 ++++++++++++++++++++++---- 8 files changed, 110 insertions(+), 17 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3993fe1..90e9977 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -446,6 +446,15 @@ All shared state lives in a single Zustand store. Key sections: `cellColorPalette`, `cellColorClamp` (squish/oob cutoffs). `cellColorType` / `cellColorCategories` hold the type the backend actually returned, written by panel 0 — the LayerPanel reads these instead of guessing from the schema dtype. + +**`usePanelSettings` ignores `viewports` / `viewportActual`.** Both are rewritten on +every OpenSeadragon viewport-change event, and no consumer of that hook reads either +— `ViewerPanel` takes `viewports[panelIndex]` through its own selector, and ⇔ Match +zoom reads `viewportActual` via `getState()`. Measured over one simulated pan (120 +writes): **121 re-renders of every sidebar section before, 0 after**, with ordinary +settings changes still delivered. Before adding a key to `IGNORED_KEYS`, check that +nothing reading it comes through the hook — ignoring a key a consumer *does* read +makes that consumer silently stale, which is far worse than a redundant render. - **Categorical override**: `categoricalOverrides`, keyed `cell::` / `edge::` → `true | false`; absent means auto-detect (issue #35). - **Metadata filters**: each is `{ field, values, min, max, includeMissing }` or null. @@ -1235,6 +1244,14 @@ cannot drift apart. `sort_categories()` sorts numerically when every label parses as a number, so cluster 10 comes after cluster 2 rather than between 1 and 2. +**A column can be present in the schema and hold nothing.** `fov` and +`transcript_count` are entirely null on the bundled MERSCOPE dataset. Such columns +now come back with `empty: True` rather than as a continuous 0–0 range, and the +filter section says "no values in this column" instead of drawing a range slider +that does nothing over a filter that correctly matches no cells. `type` is still +set, so nothing switching on categorical-vs-continuous needs a third case, and the +cross-panel merge treats a column as empty only when it is empty in *every* panel. + **`_color_values_meta` now lives on the base class.** Every reader used to carry a near-identical copy, and the six copies had already drifted — CosMx filled NaN with `""`/`0` where the others dropped it, and only some passed `key=str` to `sorted`. @@ -1409,6 +1426,15 @@ and `CACHE_DIR` relocates both the DZI pyramids and the spatial index off the da unless it is enabled anyone with the URL can view the data. There is no application-level auth, no user accounts, and no per-dataset permissions. +**Releasing.** The version lives in **three** places and they must move together — +`frontend/package.json` (read at build time via vite's `__APP_VERSION__`), +`backend/app/main.py` (`APP_VERSION`, served by `/health`), and the line under the +title in `README.md`. The sidebar badge compares the first two and turns red when +they disagree, so a half-bump is visible but only once the app is running. + +Every other `v0.x.y` in the tree is a *historical* reference — "shipped in v0.8.4", +"present since v0.2.0" — and must not be swept along by a bump. + **Regression guard.** `backend/tests/golden_snapshot.py` exercises every reader method against all local datasets, digests the results, and diffs them against a recorded baseline (238 probes across 8 datasets). Run it after any reader change: diff --git a/README.md b/README.md index 655acff..bf0b883 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # TissuePlex +**v0.8.8** + An interactive spatial transcriptomics viewer for exploring cell-cell communication from [NICHESv2](https://github.com/RaredonLab/NICHESv2) directly on the tissue image. ![TissuePlex demo](docs/demo.gif) diff --git a/backend/app/main.py b/backend/app/main.py index 5f55466..56622ba 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -3,7 +3,7 @@ from app.routers import tiles, spatial, edges, layers -APP_VERSION = "0.8.7" +APP_VERSION = "0.8.8" app = FastAPI(title="TissuePlex API", version=APP_VERSION) diff --git a/backend/app/readers/base_reader.py b/backend/app/readers/base_reader.py index 87dfd66..dcc1043 100644 --- a/backend/app/readers/base_reader.py +++ b/backend/app/readers/base_reader.py @@ -175,8 +175,18 @@ def _metadata_frame(self) -> Optional[pd.DataFrame]: def _color_values_meta(self, field: str, categorical: Optional[bool] = None) -> dict: - """Per-cell values for one metadata column, typed for the frontend.""" - empty = {"type": "continuous", "values": {}, "min": 0.0, "max": 0.0} + """Per-cell values for one metadata column, typed for the frontend. + + A column with no usable values is reported as ``empty: True`` rather than + as a continuous 0–0 range. Several real files ship such columns — + ``fov`` and ``transcript_count`` are entirely null on the bundled MERSCOPE + dataset — and the old shape gave the UI a range slider that did nothing, + a legend with no span, and a filter that correctly matched no cells while + looking broken. ``type`` is still set so nothing switching on + categorical-vs-continuous has to learn a third case. + """ + empty = {"type": "continuous", "values": {}, "min": 0.0, "max": 0.0, + "empty": True} df = self._metadata_frame() if df is None or df.empty or "cell_id" not in df.columns \ or field not in df.columns: @@ -193,6 +203,7 @@ def _color_values_meta(self, field: str, "type": "categorical", "values": values, "categories": metadata_filter.sort_categories(set(values.values())), + "empty": not values, } # to_numeric rather than float(): a forced-continuous request can land on a @@ -208,7 +219,7 @@ def _color_values_meta(self, field: str, if not finite: return empty return {"type": "continuous", "values": values, - "min": min(finite), "max": max(finite)} + "min": min(finite), "max": max(finite), "empty": False} def filter_cell_ids(self, spec: Optional[MetadataFilter]) -> Optional[set]: """Resolve a metadata filter to the set of cell ids it keeps (issue #45). diff --git a/backend/tests/edge_pipeline_check.py b/backend/tests/edge_pipeline_check.py index 3fafa6d..c3f1a4f 100644 --- a/backend/tests/edge_pipeline_check.py +++ b/backend/tests/edge_pipeline_check.py @@ -53,7 +53,10 @@ def usable_field(frame): def spec_for(reader, field): - cv = reader.color_values("metadata", None, field) + # color_values(mode, field, genes, categorical) — passing field positionally + # into `genes` silently returns the empty result, which made spec_for return + # None and skipped the subset assertion below on most datasets. + cv = reader.color_values("metadata", field) if cv.get("categories"): return MetadataFilter.build(field, values=cv["categories"][:1]) lo, hi = cv.get("min"), cv.get("max") @@ -90,7 +93,12 @@ def check(ds: str) -> str: {r["edge"] for r in er.query_structure(density=0.1)}: failures.append(f"[{ds}] sampling is not stable between identical calls") - return f"{baseline:>8,} edges filter=[{field}]" if field else f"{baseline:>8,} edges (no filterable column)" + if ids: + return f"{baseline:>8,} edges filter=[{field}] -> {len(ids):,} cells" + # Reported rather than passed over in silence: with no resolvable filter the + # subset assertion above never runs, and a check that skips its own core + # property while printing OK is worse than no check. + return f"{baseline:>8,} edges NO FILTER RESOLVED — subset assertion skipped" found = sorted(p.name for p in DATA_ROOT.iterdir() if (p / "edges.parquet").exists()) diff --git a/frontend/package.json b/frontend/package.json index 8a1fe2e..41a2203 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "tissueplex", - "version": "0.8.7", + "version": "0.8.8", "private": true, "scripts": { "dev": "vite", diff --git a/frontend/src/components/LayerPanel.jsx b/frontend/src/components/LayerPanel.jsx index 4a90166..3ea3d36 100644 --- a/frontend/src/components/LayerPanel.jsx +++ b/frontend/src/components/LayerPanel.jsx @@ -560,15 +560,20 @@ function CategoricalLegend({ field, categories = [] }) { async function mergeColorValues(promises) { const rs = (await Promise.all(promises)).filter(Boolean); if (!rs.length) return null; + // A column can be present in the schema and hold nothing — `fov` and + // `transcript_count` are entirely null on the bundled MERSCOPE dataset. Empty + // only if it is empty in *every* panel: one panel having values is enough to + // make the control worth showing. + const empty = rs.every((r) => r.empty); if (rs.some((r) => r.type === "categorical")) { const seen = new Set(), cats = []; for (const r of rs) for (const c of r.categories ?? []) if (!seen.has(c)) { seen.add(c); cats.push(c); } - return { type: "categorical", categories: cats }; + return { type: "categorical", categories: cats, empty }; } const mins = rs.map((r) => r.min).filter((v) => v != null); const maxs = rs.map((r) => r.max).filter((v) => v != null); - return { type: "continuous", min: Math.min(...mins), max: Math.max(...maxs) }; + return { type: "continuous", min: Math.min(...mins), max: Math.max(...maxs), empty }; } // ── Metadata filter (issue #45) ─────────────────────────────────────────────── @@ -632,7 +637,16 @@ function MetadataFilterSection({
loading values…
)} - {field && !loading && meta?.type === "categorical" && ( + {/* Present in the schema but holding nothing. Saying so beats a range + slider that spans 0–0 and a filter that correctly matches no cells + while looking broken. */} + {field && !loading && meta?.empty && ( +
+ no values in this column +
+ )} + + {field && !loading && !meta?.empty && meta?.type === "categorical" && (
{(meta.categories ?? []).map((cat) => ( @@ -670,7 +684,7 @@ function MetadataFilterSection({
)} - {field && !loading && meta?.type === "continuous" && ( + {field && !loading && !meta?.empty && meta?.type === "continuous" && (
min s, equalIgnoringHotKeys); const settings = store.panels[i]?.settings ?? store.panels[0].settings; return { ...store, ...settings, panelIndex: i }; } From 67365ccd69b6803cda6f1f7acc6d1c2fcf7f4799 Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Tue, 18 Aug 2026 12:03:01 -0400 Subject: [PATCH 15/15] chore: v0.8.8 Co-Authored-By: Claude Opus 5