Deduplicate entries database on raw input

database/scripts/load_bgcs.py

@@ -12,9 +12,16 @@
 provide (MIBiG 4.0's GBKs dropped the ACCESSION.VERSION suffix) -- so it's
 looked up here instead, from the same JSON-derived mibig_versions.json
 extract_mibig_compounds.py produces for load_compounds.py.
+
+BGCs are deduplicated per source .gbk file via `file_hash` (parse_gbks.py's
+sha256 of the whole file's raw text): a region's entry id is derived from that
+hash plus its own readout id, and a file whose hash is already stored (checked
+via RetroMolDuckDB.bgc_content_hash_exists) is skipped outright, so rerunning
+the pipeline over an already-ingested file doesn't redo any fingerprinting work.
 """
 
 import argparse
+import hashlib
 import json
 import logging
 from pathlib import Path
@@ -43,6 +50,7 @@ def run(
 
     added = 0
     skipped = 0
+    skipped_existing_file = 0
 
     db = RetroMolDuckDB.open(db_path)
     try:
@@ -53,6 +61,12 @@ def run(
                     continue
 
                 entry = json.loads(line)
+                file_hash = entry.get("file_hash")
+
+                if file_hash and db.bgc_content_hash_exists(file_hash):
+                    skipped_existing_file += 1
+                    continue
+
                 readout = LinearReadout.from_dict(entry["readout"])
                 names, tokens = bgc_primary_sequence(readout, ruleset)
 
@@ -64,20 +78,27 @@ def run(
                 accession = entry.get("accession")
                 name = f"{accession} ({readout.id})" if accession else readout.id
                 url = mibig_url(accession, versions.get(accession)) if accession else None
+                entry_id = hashlib.sha256(f"{file_hash}:{readout.id}".encode("utf-8")).hexdigest()
 
                 db.add_entry(
+                    entry_id=entry_id,
                     name=name,
+                    database_name="MIBiG",
                     url=url,
                     raw=entry.get("raw_gbk") if include_raw_gbk else None,
                     entry_type="bgc",
                     primary_sequence=names,
                     fingerprint=fp,
+                    content_hash=file_hash,
                 )
                 added += 1
     finally:
         db.close()
 
-    log.info("load_bgcs: added=%d skipped=%d", added, skipped)
+    log.info(
+        "load_bgcs: added=%d skipped=%d skipped_existing_file=%d",
+        added, skipped, skipped_existing_file,
+    )
 
 
 def main() -> None:

database/scripts/load_compounds.py

@@ -4,8 +4,13 @@
 compound's tailoring events (glycosylation, methylation -- anything that shows up as
 its own disconnected single-node path) -- is merged into the one primary sequence
 stored for that compound: longest path first, ties broken lexicographically, joined
-by TOKEN_LINK (see common.primary_sequence_from_result). One compound -> one db
-entry, always. `raw` is the original input SMILES.
+by TOKEN_LINK (see common.primary_sequence_from_result). `raw` is the original input
+SMILES.
+
+Compounds are deduplicated on `result.submission.inchikey` (stereo-aware, computed by
+retromol.model.submission.Submission with keep_stereo=True). The same molecule
+appearing in both NPAtlas and MIBiG lands as one database entry with two source
+records (see RetroMolDuckDB.add_entry) rather than two separate rows.
 """
 
 import argparse
@@ -31,6 +36,8 @@
 
 log = logging.getLogger(__name__)
 
+DATABASE_NAMES = {"npatlas": "NPAtlas", "mibig": "MIBiG"}
+
 
 def _npatlas_name_and_url(props: dict) -> tuple[str | None, str | None]:
     npaid_key = find_key_ci(props, ["npaid"])
@@ -102,7 +109,9 @@ def run(
                         fp = fingerprinter.encode(tokens)
 
                         db.add_entry(
+                            entry_id=result.submission.inchikey,
                             name=name,
+                            database_name=DATABASE_NAMES[source],
                             url=url,
                             raw=result.submission.smiles,
                             entry_type="compound",

database/scripts/parse_gbks.py

@@ -29,6 +29,7 @@
 """
 
 import argparse
+import hashlib
 import itertools
 import json
 import logging
@@ -86,6 +87,7 @@ def _process_file(path_str: str) -> tuple[list[dict], str | None]:
     path = Path(path_str)
     try:
         raw_gbk = path.read_text()
+        file_hash = hashlib.sha256(raw_gbk.encode("utf-8")).hexdigest()
         regions = parse_antismash_gbk(path, AntiSmashOptions())
 
         entries: list[dict] = []
@@ -105,6 +107,7 @@ def _process_file(path_str: str) -> tuple[list[dict], str | None]:
                 "accession": accession,
                 "file_name": region.file_name,
                 "raw_gbk": raw_gbk,
+                "file_hash": file_hash,
                 "readout": readout.to_dict(),
             })
 

gui/src/client/src/components/workspace/AlignmentGrid.tsx

@@ -12,7 +12,6 @@ import Collapse from "@mui/material/Collapse";
 import Stack from "@mui/material/Stack";
 import Tooltip from "@mui/material/Tooltip";
 import Typography from "@mui/material/Typography";
-import MuiLink from "@mui/material/Link";
 import ExpandMoreIcon from "@mui/icons-material/ExpandMore";
 import ExpandLessIcon from "@mui/icons-material/ExpandLess";
 import DownloadIcon from "@mui/icons-material/Download";
@@ -236,7 +235,6 @@ export function ResultRow({
   sendingToUploads?: boolean;
 }) {
   const [expanded, setExpanded] = React.useState(false);
-  const theme = useTheme();
 
   const canOfferSend = result.origin === "database" && !!onSendToUploads;
   const hasRaw = !!result.raw;
@@ -262,22 +260,9 @@ export function ResultRow({
         </Typography>
 
         <Box sx={{ flex: 1, minWidth: 0 }}>
-          {result.url ? (
-            <MuiLink
-              href={result.url}
-              target="_blank"
-              rel="noopener noreferrer"
-              underline="hover"
-              color={(theme.vars || theme).palette.primary.main}
-              sx={{ fontWeight: 500 }}
-            >
-              {result.name}
-            </MuiLink>
-          ) : (
-            <Typography variant="body2" fontWeight={500} noWrap>
-              {result.name}
-            </Typography>
-          )}
+          <Typography variant="body2" fontWeight={500} noWrap>
+            {result.name}
+          </Typography>
         </Box>
 
         <Chip
@@ -316,6 +301,32 @@ export function ResultRow({
 
       <Collapse in={expanded} unmountOnExit>
         <Box sx={{ mt: 1, pl: 4.5 }}>
+          {result.sources.length > 0 && (
+            <Stack direction="row" spacing={1} sx={{ mb: 1, flexWrap: "wrap", gap: 1 }}>
+              {result.sources.map((source, idx) =>
+                source.url ? (
+                  <Chip
+                    key={idx}
+                    component="a"
+                    href={source.url}
+                    target="_blank"
+                    rel="noopener noreferrer"
+                    clickable
+                    size="small"
+                    variant="outlined"
+                    label={`${source.databaseName}: ${source.name}`}
+                  />
+                ) : (
+                  <Chip
+                    key={idx}
+                    size="small"
+                    variant="outlined"
+                    label={`${source.databaseName}: ${source.name}`}
+                  />
+                )
+              )}
+            </Stack>
+          )}
           <AlignmentGrid
             rows={[
               { id: "query", label: "Query", sequence: result.alignedQuery },

gui/src/client/src/components/workspace/WorkspaceDiscovery.tsx

@@ -643,23 +643,23 @@ export const WorkspaceDiscovery: React.FC<WorkspaceDiscoveryProps> = ({ session,
                         display: "flex",
                         alignItems: "center",
                         gap: 1.5,
-                        flexWrap: "wrap",
+                        minWidth: 0,
                       }}
                     >
                       <Typography variant="caption" color="text.secondary" sx={{ flexShrink: 0 }}>
                         {`reconstructed backbone ${idx+1}`}
                       </Typography>
                       {override && (
-                        <Chip label="Edited" size="small" color="info" variant="outlined" sx={{ fontSize: "0.7rem" }} />
+                        <Chip label="Edited" size="small" color="info" variant="outlined" sx={{ fontSize: "0.7rem", flexShrink: 0 }} />
                       )}
-                      <Box sx={{ transform: "translateY(5px)" }}>
+                      <Box sx={{ flex: 1, minWidth: 0, transform: "translateY(5px)" }}>
                         <ReconstructionPreview sequence={effectiveSequence} />
                       </Box>
                       <Button
                         size="small"
                         variant="outlined"
                         onClick={() => handlePickNames(effectiveSequence.map(([name]) => name))}
-                        sx={{ ml: "auto" }}
+                        sx={{ flexShrink: 0 }}
                       >
                         Use this
                       </Button>
@@ -695,20 +695,20 @@ export const WorkspaceDiscovery: React.FC<WorkspaceDiscoveryProps> = ({ session,
                       display: "flex",
                       alignItems: "center",
                       gap: 1.5,
-                      flexWrap: "wrap",
+                      minWidth: 0,
                     }}
                   >
                     <Typography variant="caption" color="text.secondary" sx={{ flexShrink: 0 }}>
                       {region.id}
                     </Typography>
-                    <Box sx={{ transform: "translateY(5px)" }}>
+                    <Box sx={{ flex: 1, minWidth: 0, transform: "translateY(5px)" }}>
                       <NamesPreview names={region.primary_sequence} />
                     </Box>
                     <Button
                       size="small"
                       variant="outlined"
                       onClick={() => handlePickNames(region.primary_sequence)}
-                      sx={{ ml: "auto" }}
+                      sx={{ flexShrink: 0 }}
                     >
                       Use this
                     </Button>

gui/src/client/src/features/discovery/types.ts

@@ -85,6 +85,11 @@ export const DiscoveryResultSchema = z.object({
   // database entry can be resubmitted for parsing under the current rule set. Null for
   // upload-origin BGC candidates and for any entry that predates `raw` being stored.
   raw: z.string().nullable().default(null),
+  // Every (name, source database) this entry is known under -- e.g. a compound present
+  // in both NPAtlas and MIBiG carries two entries here. Empty for upload-origin results.
+  sources: z
+    .array(z.object({ name: z.string(), databaseName: z.string(), url: z.string().nullable() }))
+    .default([]),
   fingerprintSimilarity: z.number(),
   primarySequence: z.array(z.string()),
   inverted: z.boolean(),

gui/src/server/routes/discovery.py

@@ -720,6 +720,14 @@ def rank_key(align_score: float, target_self_score: float | None) -> float:
             # offer "send back to Uploads" so a retrieved entry can be reparsed under
             # the current rule set without the user having to re-supply it by hand.
             "raw": candidate.entry.raw,
+            # Every (name, database) this entry is known under -- e.g. a compound
+            # present in both NPAtlas and MIBiG carries two sources. Empty for
+            # upload-origin candidates, which aren't backed by any source database
+            # (see Entry.sources / duckdb.py).
+            "sources": [
+                {"name": s.name, "databaseName": s.database_name, "url": s.url}
+                for s in candidate.entry.sources
+            ],
             "fingerprintSimilarity": candidate.similarity,
             "primarySequence": [_denormalize_for_display(t) for t in oriented_target],
             "inverted": inverted,

gui/src/server/routes/jobs.py

@@ -97,22 +97,23 @@ def search_compound_by_name():
             rows = db.con.execute(
                 """
                 SELECT
-                    min(id) AS id,
-                    name,
-                    url,
-                    raw
-                FROM entries
-                WHERE type = 'compound'
-                    AND raw IS NOT NULL
-                    AND lower(name) LIKE lower(?)
-                GROUP BY name, url, raw
+                    es.entry_id AS id,
+                    es.name AS name,
+                    es.database_name AS database_name,
+                    es.url AS url,
+                    e.raw AS raw
+                FROM entry_sources es
+                JOIN entries e ON e.id = es.entry_id
+                WHERE e.type = 'compound'
+                    AND e.raw IS NOT NULL
+                    AND lower(es.name) LIKE lower(?)
                 ORDER BY
                     CASE
-                        WHEN lower(name) = lower(?) THEN 0
-                        WHEN lower(name) LIKE lower(?) THEN 1
+                        WHEN lower(es.name) = lower(?) THEN 0
+                        WHEN lower(es.name) LIKE lower(?) THEN 1
                         ELSE 2
                     END,
-                    name
+                    es.name
                 LIMIT ?
                 """,
                 [like, q, f"{q}%", limit],
@@ -126,11 +127,11 @@ def search_compound_by_name():
         {
             "name": name,
             "smiles": raw,
-            "databaseName": "RetroMol",
+            "databaseName": database_name,
             "databaseIdentifier": entry_id,
             "url": url,
         }
-        for entry_id, name, url, raw in rows
+        for entry_id, name, database_name, url, raw in rows
         if name and raw
     ]