From 5a1429a624a80f00fc3d9579bcd58d54c5b66584 Mon Sep 17 00:00:00 2001
From: Ashu <ashutosh@root.credit>
Date: Wed, 1 Jul 2026 12:16:35 -0700
Subject: [PATCH 01/15] add who guideline scraper

---
 .../amfv_datasets/scraping/LICENSE_NOTES.md   |  30 ++
 datasets/amfv_datasets/scraping/__init__.py   |  18 +
 datasets/amfv_datasets/scraping/cli.py        |   6 +-
 datasets/amfv_datasets/scraping/who.py        | 379 ++++++++++++++++++
 datasets/test/fixtures/who_listing_api.json   |  20 +
 .../fixtures/who_publication_overview.html    |  17 +
 datasets/test/test_scraping_who.py            | 155 +++++++
 7 files changed, 624 insertions(+), 1 deletion(-)
 create mode 100644 datasets/amfv_datasets/scraping/LICENSE_NOTES.md
 create mode 100644 datasets/amfv_datasets/scraping/who.py
 create mode 100644 datasets/test/fixtures/who_listing_api.json
 create mode 100644 datasets/test/fixtures/who_publication_overview.html
 create mode 100644 datasets/test/test_scraping_who.py

diff --git a/datasets/amfv_datasets/scraping/LICENSE_NOTES.md b/datasets/amfv_datasets/scraping/LICENSE_NOTES.md
new file mode 100644
index 00000000..a98b822b
--- /dev/null
+++ b/datasets/amfv_datasets/scraping/LICENSE_NOTES.md
@@ -0,0 +1,30 @@
+# Source licensing notes for scraped corpora
+
+## WHO (World Health Organization)
+
+Publications on [who.int](https://www.who.int/publications) published since
+November 2016 are licensed under **Creative Commons Attribution-NonCommercial-
+ShareAlike 3.0 IGO** (CC BY-NC-SA 3.0 IGO).
+
+- **Non-commercial use and adaptation** are permitted.
+- **Attribution** to WHO is required.
+- **Share-alike**: derivatives must use the same or a similar licence.
+
+Pre-2017 publications were not reissued under this licence. Use
+`metadata.publication_date` to filter when building corpora.
+
+### Suggested attribution
+
+> © World Health Organization {year}. *{publication title}*.
+> Licensed under CC BY-NC-SA 3.0 IGO.
+> https://creativecommons.org/licenses/by-nc-sa/3.0/igo/
+
+Each scraped document also records `metadata.license` and
+`metadata.attribution`.
+
+### Milestone 1 content scope
+
+The WHO scraper collects the HTML **Overview** section from each publication
+landing page (`metadata.content_scope = "overview"`). Full guideline text is
+typically available only as a linked PDF (`metadata.download_url`). PDF
+extraction is intentionally deferred.
diff --git a/datasets/amfv_datasets/scraping/__init__.py b/datasets/amfv_datasets/scraping/__init__.py
index cd7cdb06..110f6093 100644
--- a/datasets/amfv_datasets/scraping/__init__.py
+++ b/datasets/amfv_datasets/scraping/__init__.py
@@ -27,6 +27,16 @@
     scrape_guideline,
     scrape_nice,
 )
+from amfv_datasets.scraping.who import (
+    WhoFetchError,
+    WhoListingPage,
+    WhoPublicationRef,
+    build_publication_text,
+    list_publications,
+    publication_ref_from_url,
+    scrape_publication,
+    scrape_who,
+)
 
 __all__ = [
     "GuidanceRef",
@@ -39,8 +49,12 @@
     "ScrapedDocument",
     "ScraperSource",
     "USER_AGENT",
+    "WhoFetchError",
+    "WhoListingPage",
+    "WhoPublicationRef",
     "absolute_unique_urls",
     "build_guideline_text",
+    "build_publication_text",
     "clean_text",
     "document_title",
     "default_client",
@@ -48,7 +62,11 @@
     "guidance_ref_from_url",
     "html_to_markdown",
     "list_published_guidance",
+    "list_publications",
+    "publication_ref_from_url",
     "scrape_guideline",
     "scrape_listing_documents",
     "scrape_nice",
+    "scrape_publication",
+    "scrape_who",
 ]
diff --git a/datasets/amfv_datasets/scraping/cli.py b/datasets/amfv_datasets/scraping/cli.py
index e8aece34..59e69bd7 100644
--- a/datasets/amfv_datasets/scraping/cli.py
+++ b/datasets/amfv_datasets/scraping/cli.py
@@ -27,6 +27,7 @@
 from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun
 from amfv_datasets.scraping.html import LinkMode
 from amfv_datasets.scraping.nice import scrape_nice
+from amfv_datasets.scraping.who import scrape_who
 
 
 class ScraperSource(StrEnum):
@@ -34,6 +35,7 @@ class ScraperSource(StrEnum):
 
     ALL = "all"
     NICE = "nice"
+    WHO = "who"
 
 
 class OutputFormat(StrEnum):
@@ -72,6 +74,8 @@ def scrape_documents(
         match selected_source:
             case ScraperSource.NICE:
                 return scrape_nice(documents=documents, link_mode=link_mode, url=url)
+            case ScraperSource.WHO:
+                return scrape_who(documents=documents, link_mode=link_mode, url=url)
             case ScraperSource.ALL:
                 raise AssertionError("expanded source cannot be all")
     raise AssertionError(f"unsupported source: {source}")
@@ -125,7 +129,7 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path
 
 def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]:
     if source is ScraperSource.ALL:
-        return (ScraperSource.NICE,)
+        return (ScraperSource.NICE, ScraperSource.WHO)
     return (source,)
 
 
diff --git a/datasets/amfv_datasets/scraping/who.py b/datasets/amfv_datasets/scraping/who.py
new file mode 100644
index 00000000..a2412511
--- /dev/null
+++ b/datasets/amfv_datasets/scraping/who.py
@@ -0,0 +1,379 @@
+"""Scrape WHO guideline publications into normalized markdown documents.
+
+WHO (World Health Organization) publishes guidelines primarily as PDFs hosted on
+iris.who.int. The publication landing pages expose a short HTML Overview section
+plus bibliographic metadata. For Milestone 1 we scrape that Overview text rather
+than PDFs, because PDF extraction loses reading order and interleaves
+headers/footers. Full-text PDF extraction is intentionally deferred.
+
+Discovery uses WHO's Sitefinity OData publications hub API, filtered to the
+Guidelines publishing office. Extraction fetches each publication HTML page and
+converts the Overview block to markdown.
+
+Attribution:
+The publishing-office filter UUID and PDF-first approach in Meditron's WHO
+scraper (epfLLM/meditron, gap-replay/guidelines/scrapers/scrapers.py) informed
+discovery scope. Source license: Apache License 2.0.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import urlparse
+
+import httpx
+from lxml import html as lxml_html
+
+from amfv_datasets.scraping.base import (
+    ScrapedDocument,
+    ScrapeError,
+    ScrapeRun,
+    default_client,
+    scrape_listing_documents,
+)
+from amfv_datasets.scraping.html import LinkMode, clean_text, document_title, html_to_markdown
+
+BASE_URL = "https://www.who.int"
+GUIDELINES_LISTING_URL = f"{BASE_URL}/publications/who-guidelines"
+GUIDELINES_PUBLISHING_OFFICE = "c09761c0-ab8e-4cfa-9744-99509c4d306b"
+SF_SITE = "15210d59-ad60-47ff-a542-7ed76645f0c7"
+PUBLICATIONS_API_PATH = "/api/hubs/publications"
+LISTING_PAGE_SIZE = 25
+DOCUMENT_DELAY_SECONDS = 5.0
+WHO_LICENSE = "CC BY-NC-SA 3.0 IGO"
+WHO_ATTRIBUTION = "© World Health Organization. Licensed under CC BY-NC-SA 3.0 IGO."
+CONTENT_SCOPE = "overview"
+
+logger = logging.getLogger(__name__)
+
+_PUBLICATION_PATH_RE = re.compile(
+    r"^/publications/i/item/(?P<publication_id>[^/?#]+)(?:/|$)",
+    re.IGNORECASE,
+)
+_ISBN_RE = re.compile(r"ISBN:\s*([\d\-]+)", re.IGNORECASE)
+
+class WhoFetchError(ScrapeError):
+    """Raised when a WHO publication cannot be fetched or parsed."""
+
+
+@dataclass(frozen=True)
+class WhoPublicationRef:
+    """A WHO guideline publication reference from the listing."""
+
+    publication_id: str
+    title: str
+    page_url: str
+    publication_date: str | None = None
+    tag: str | None = None
+    download_url: str | None = None
+
+
+@dataclass(frozen=True)
+class WhoListingPage:
+    """A page of WHO guideline publication references."""
+
+    refs: list[WhoPublicationRef]
+    total: int | None
+
+
+def publication_ref_from_url(url: str) -> WhoPublicationRef:
+    """Parse a WHO publication URL into a canonical publication reference.
+
+    Args:
+        url: WHO publication URL to parse.
+    """
+    parsed = urlparse(url.strip())
+    if parsed.scheme not in {"http", "https"} or parsed.netloc.lower() not in {
+        "www.who.int",
+        "who.int",
+    }:
+        raise WhoFetchError(f"Enter a WHO publication URL from who.int; got {url!r}")
+
+    match = _PUBLICATION_PATH_RE.match(parsed.path.rstrip("/") + "/")
+    if not match:
+        raise WhoFetchError(
+            f"Enter a URL like https://www.who.int/publications/i/item/9789240121805; got {url!r}"
+        )
+
+    publication_id = match.group("publication_id")
+    return WhoPublicationRef(
+        publication_id=publication_id,
+        title=publication_id,
+        page_url=_page_url(publication_id=publication_id),
+    )
+
+
+def _page_url(*, publication_id: str) -> str:
+    return f"{BASE_URL}/publications/i/item/{publication_id}"
+
+
+def _publications_api_params(*, page: int) -> dict[str, str]:
+    skip = (page - 1) * LISTING_PAGE_SIZE
+    return {
+        "sf_site": SF_SITE,
+        "sf_provider": "OpenAccessProvider",
+        "sf_culture": "en",
+        "$orderby": "PublicationDateAndTime desc",
+        "$select": "Title,ItemDefaultUrl,FormatedDate,Tag,DownloadUrl",
+        "$filter": f"publishingoffices/any(s:s eq {GUIDELINES_PUBLISHING_OFFICE})",
+        "$top": str(LISTING_PAGE_SIZE),
+        "$skip": str(skip),
+        "$count": "true",
+    }
+
+
+def _publication_id_from_item_url(item_default_url: str) -> str:
+    return item_default_url.strip("/").split("/")[-1]
+
+
+def _parse_api_listing(payload: dict[str, Any]) -> WhoListingPage:
+    """Parse a WHO publications OData response into listing refs."""
+    try:
+        items = payload["value"]
+    except KeyError as exc:
+        raise WhoFetchError("Could not parse WHO publications API JSON") from exc
+
+    refs: list[WhoPublicationRef] = []
+    for item in items:
+        item_default_url = (item.get("ItemDefaultUrl") or "").strip()
+        title = (item.get("Title") or "").strip()
+        if not item_default_url or not title:
+            continue
+        publication_id = _publication_id_from_item_url(item_default_url)
+        refs.append(
+            WhoPublicationRef(
+                publication_id=publication_id,
+                title=title,
+                page_url=_page_url(publication_id=publication_id),
+                publication_date=(item.get("FormatedDate") or None),
+                tag=(item.get("Tag") or None),
+                download_url=(item.get("DownloadUrl") or None),
+            )
+        )
+
+    total = payload.get("@odata.count")
+    return WhoListingPage(refs=refs, total=total if isinstance(total, int) else None)
+
+
+def _parse_html_listing(html_text: str) -> WhoListingPage:
+    """Parse server-rendered WHO guideline cards from the listing page."""
+    doc = lxml_html.fromstring(html_text)
+    refs: list[WhoPublicationRef] = []
+    seen: set[str] = set()
+    for link in doc.xpath("//a[contains(@href,'/publications/i/item/')]"):
+        href = link.get("href", "").split("?")[0]
+        match = _PUBLICATION_PATH_RE.match(href.rstrip("/") + "/")
+        if not match or href in seen:
+            continue
+        seen.add(href)
+        publication_id = match.group("publication_id")
+        title = clean_text(link.xpath("string(.)")) or publication_id
+        refs.append(
+            WhoPublicationRef(
+                publication_id=publication_id,
+                title=title,
+                page_url=_page_url(publication_id=publication_id),
+            )
+        )
+    return WhoListingPage(refs=refs, total=None)
+
+
+def list_publications(client: httpx.Client, page: int = 1) -> WhoListingPage:
+    """Return one page of WHO guideline publication refs.
+
+    Args:
+        client: HTTP client used to fetch the publications API.
+        page: Listing page number (default: 1).
+    """
+    response = client.get(f"{BASE_URL}{PUBLICATIONS_API_PATH}", params=_publications_api_params(page=page))
+    if response.status_code >= 400:
+        logger.warning("WHO publications API returned %s; falling back to HTML listing", response.status_code)
+        listing = client.get(GUIDELINES_LISTING_URL)
+        listing.raise_for_status()
+        return _parse_html_listing(listing.text)
+
+    try:
+        payload = response.json()
+    except json.JSONDecodeError as exc:
+        raise WhoFetchError("Could not decode WHO publications API JSON") from exc
+    return _parse_api_listing(payload)
+
+
+def _overview_html(section: lxml_html.HtmlElement) -> str:
+    """Return HTML for the Overview block within a publication section."""
+    overview_headings = section.xpath(".//h3[normalize-space()='Overview']")
+    if not overview_headings:
+        return lxml_html.tostring(section, encoding="unicode")
+
+    heading = overview_headings[0]
+    parts = [lxml_html.tostring(heading, encoding="unicode")]
+    sibling = heading.getnext()
+    while sibling is not None and sibling.tag.lower() not in {"h2", "h3"}:
+        parts.append(lxml_html.tostring(sibling, encoding="unicode"))
+        sibling = sibling.getnext()
+    return "".join(parts)
+
+
+def _page_isbn(html_text: str) -> str | None:
+    doc = lxml_html.fromstring(html_text)
+    for node in doc.xpath("//*[contains(normalize-space(.), 'ISBN')]"):
+        match = _ISBN_RE.search(node.text_content())
+        if match:
+            return match.group(1)
+    return None
+
+
+def _page_publication_date(doc: lxml_html.HtmlElement) -> str | None:
+    values = doc.xpath("//*[contains(@class,'dynamic-content__date')]/text()")
+    return clean_text(values[0]) if values else None
+
+
+def _page_tag(doc: lxml_html.HtmlElement) -> str | None:
+    values = doc.xpath("//*[contains(@class,'dynamic-content__tag')]/text()")
+    text = clean_text(" ".join(values))
+    return text.lstrip("| ").strip() if text else None
+
+
+def _page_download_url(doc: lxml_html.HtmlElement) -> str | None:
+    for href in doc.xpath("//a[contains(@href,'iris.who.int')]/@href"):
+        if "bitstreams" in href:
+            return href.split("?")[0]
+    return None
+
+
+def build_publication_text(
+    client: httpx.Client,
+    ref: WhoPublicationRef,
+    *,
+    link_mode: LinkMode = LinkMode.KEEP,
+) -> tuple[str, int, str, dict[str, Any]]:
+    """Scrape a publication Overview into markdown text and bibliographic metadata.
+
+    Args:
+        client: HTTP client used to fetch the publication page.
+        ref: WHO publication reference to scrape.
+        link_mode: Whether links are kept as markdown links or stripped to their
+            visible text (default: LinkMode.KEEP).
+    """
+    response = client.get(ref.page_url)
+    response.raise_for_status()
+    html_text = response.text
+    doc = lxml_html.fromstring(html_text)
+
+    sections = doc.xpath("//section[contains(@class,'dynamic-content__section')]")
+    if not sections:
+        raise WhoFetchError(f"No publication content section for '{ref.publication_id}'")
+
+    title = (
+        ref.title
+        if ref.title != ref.publication_id
+        else document_title(html_text, fallback=ref.publication_id)
+    )
+    overview_html = _overview_html(sections[0])
+    markdown = html_to_markdown(overview_html, link_mode=link_mode, base_url=BASE_URL).strip()
+    if not markdown:
+        raise WhoFetchError(f"No readable Overview content for '{ref.publication_id}'")
+
+    extra_metadata: dict[str, Any] = {
+        "publication_date": ref.publication_date or _page_publication_date(doc),
+        "tag": ref.tag or _page_tag(doc),
+        "isbn": _page_isbn(html_text),
+        "download_url": ref.download_url or _page_download_url(doc),
+        "content_scope": CONTENT_SCOPE,
+        "license": WHO_LICENSE,
+        "attribution": WHO_ATTRIBUTION,
+        "listing_category": "who-guidelines",
+    }
+    return markdown, 1, title, extra_metadata
+
+
+def scrape_publication(
+    client: httpx.Client,
+    ref: WhoPublicationRef,
+    *,
+    link_mode: LinkMode = LinkMode.KEEP,
+) -> ScrapedDocument:
+    """Scrape a WHO publication into a normalized document.
+
+    Args:
+        client: HTTP client used to fetch the publication page.
+        ref: WHO publication reference to scrape.
+        link_mode: Whether links are kept as markdown links or stripped to their
+            visible text (default: LinkMode.KEEP).
+    """
+    content, section_count, title, metadata = build_publication_text(client, ref, link_mode=link_mode)
+    return ScrapedDocument(
+        source="who",
+        external_id=f"who-{ref.publication_id}",
+        title=title,
+        url=ref.page_url,
+        content=content,
+        section_count=section_count,
+        metadata={
+            "publication_id": ref.publication_id,
+            **metadata,
+        },
+    )
+
+
+def scrape_who(
+    *,
+    documents: int | None,
+    link_mode: LinkMode = LinkMode.KEEP,
+    url: str | None = None,
+) -> ScrapeRun:
+    """Scrape WHO documents from a URL or guideline listing pages.
+
+    Args:
+        documents: Number of documents to scrape. Ignored when `url` is set.
+            When unset, WHO listing pages are fetched until a page returns no
+            items (default: None).
+        link_mode: Whether links are kept as markdown links or stripped to their
+            visible text (default: LinkMode.KEEP).
+        url: WHO publication URL to scrape as a single document (default: None).
+    """
+    if url is not None:
+
+        def scrape_url() -> Iterable[ScrapedDocument]:
+            with default_client() as client:
+                yield scrape_publication(client, publication_ref_from_url(url), link_mode=link_mode)
+
+        return ScrapeRun(documents=scrape_url(), total=1)
+
+    with default_client() as client:
+        first_page = list_publications(client, page=1)
+    total = first_page.total if documents is None or first_page.total is None else min(documents, first_page.total)
+    return ScrapeRun(
+        total=total,
+        documents=scrape_listing_documents(
+            documents=documents,
+            client_factory=default_client,
+            first_page_items=first_page.refs,
+            list_page=lambda client, page: list_publications(client, page).refs,
+            scrape_item=lambda client, ref: scrape_publication(client, ref, link_mode=link_mode),
+            document_delay_seconds=DOCUMENT_DELAY_SECONDS,
+        ),
+    )
+
+
+__all__ = [
+    "BASE_URL",
+    "CONTENT_SCOPE",
+    "DOCUMENT_DELAY_SECONDS",
+    "GUIDELINES_LISTING_URL",
+    "WHO_ATTRIBUTION",
+    "WHO_LICENSE",
+    "WhoFetchError",
+    "WhoListingPage",
+    "WhoPublicationRef",
+    "build_publication_text",
+    "list_publications",
+    "publication_ref_from_url",
+    "scrape_publication",
+    "scrape_who",
+]
diff --git a/datasets/test/fixtures/who_listing_api.json b/datasets/test/fixtures/who_listing_api.json
new file mode 100644
index 00000000..a4392d3b
--- /dev/null
+++ b/datasets/test/fixtures/who_listing_api.json
@@ -0,0 +1,20 @@
+{
+  "@odata.context": "https://www.who.int/api/hubs/$metadata#publications(Title,ItemDefaultUrl,FormatedDate,Tag,DownloadUrl)",
+  "@odata.count": 356,
+  "value": [
+    {
+      "ItemDefaultUrl": "/9789240121805",
+      "Title": "Guidelines for the prevention of bloodstream infections and other infections associated with the use of intravascular catheters: part 2: central venous catheters",
+      "FormatedDate": "28 May 2026",
+      "Tag": "Guideline",
+      "DownloadUrl": "https://iris.who.int/server/api/core/bitstreams/f750f24d-c0c2-425c-85fd-ec310d2ce994/content"
+    },
+    {
+      "ItemDefaultUrl": "/9789240121744",
+      "Title": "WHO guideline for screening and treatment of cervical pre-cancer lesions for cervical cancer prevention",
+      "FormatedDate": "8 May 2026",
+      "Tag": "Guideline",
+      "DownloadUrl": "https://iris.who.int/server/api/core/bitstreams/32214b73-0e95-4243-9e83-617948510dcd/content"
+    }
+  ]
+}
diff --git a/datasets/test/fixtures/who_publication_overview.html b/datasets/test/fixtures/who_publication_overview.html
new file mode 100644
index 00000000..b6f16f29
--- /dev/null
+++ b/datasets/test/fixtures/who_publication_overview.html
@@ -0,0 +1,17 @@
+<html>
+  <article class="dynamic-content dynamic-content__article dynamic-content__publication">
+    <h1>Guidelines for the prevention of bloodstream infections and other infections associated with the use of intravascular catheters: part 2: central venous catheters</h1>
+    <div class="dynamic-content__data">
+      <div class="dynamic-content__date">28 May 2026</div>
+      <div class="dynamic-content__tag"><span>&nbsp;&#124;&nbsp;</span>Guideline</div>
+    </div>
+    <section class="dynamic-content__section dynamic-content__container">
+      <a href="https://iris.who.int/server/api/core/bitstreams/f750f24d-c0c2-425c-85fd-ec310d2ce994/content">Download (1.2 MB)</a>
+      <h3>Overview</h3>
+      <p>These WHO guidelines provide evidence-based recommendations for the prevention of bloodstream infections (BSIs) and other infections associated with the use of central venous catheters (CVCs) across health care settings.</p>
+      <h3>WHO Team</h3>
+      <p>Infection Prevention and Control (IPC)</p>
+      <p><strong>ISBN:</strong> 978-92-4-012180-5</p>
+    </section>
+  </article>
+</html>
diff --git a/datasets/test/test_scraping_who.py b/datasets/test/test_scraping_who.py
new file mode 100644
index 00000000..d3db37b3
--- /dev/null
+++ b/datasets/test/test_scraping_who.py
@@ -0,0 +1,155 @@
+"""Tests for WHO scraping helpers."""
+
+import json
+from pathlib import Path
+
+import httpx
+import pytest
+
+from amfv_datasets.scraping.html import LinkMode
+from amfv_datasets.scraping.who import (
+    BASE_URL,
+    WHO_LICENSE,
+    WhoFetchError,
+    WhoListingPage,
+    WhoPublicationRef,
+    build_publication_text,
+    list_publications,
+    publication_ref_from_url,
+    scrape_publication,
+)
+
+_FIXTURES = Path(__file__).parent / "fixtures"
+_CVC_GUIDELINE_TITLE = (
+    "Guidelines for the prevention of bloodstream infections and other infections "
+    "associated with the use of intravascular catheters: part 2: central venous catheters"
+)
+_CERVICAL_GUIDELINE_TITLE = (
+    "WHO guideline for screening and treatment of cervical pre-cancer lesions "
+    "for cervical cancer prevention"
+)
+
+
+def test_list_publications_parses_api_listing() -> None:
+    """WHO listing payloads are parsed from the publications OData API."""
+    payload = json.loads((_FIXTURES / "who_listing_api.json").read_text(encoding="utf-8"))
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert request.url.path == "/api/hubs/publications"
+        assert request.url.params["$filter"] == "publishingoffices/any(s:s eq c09761c0-ab8e-4cfa-9744-99509c4d306b)"
+        assert request.url.params["$skip"] == "0"
+        return httpx.Response(200, json=payload)
+
+    client = httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL)
+
+    listing_page = list_publications(client)
+
+    assert listing_page == WhoListingPage(
+        total=356,
+        refs=[
+            WhoPublicationRef(
+                publication_id="9789240121805",
+                title=_CVC_GUIDELINE_TITLE,
+                page_url="https://www.who.int/publications/i/item/9789240121805",
+                publication_date="28 May 2026",
+                tag="Guideline",
+                download_url="https://iris.who.int/server/api/core/bitstreams/f750f24d-c0c2-425c-85fd-ec310d2ce994/content",
+            ),
+            WhoPublicationRef(
+                publication_id="9789240121744",
+                title=_CERVICAL_GUIDELINE_TITLE,
+                page_url="https://www.who.int/publications/i/item/9789240121744",
+                publication_date="8 May 2026",
+                tag="Guideline",
+                download_url="https://iris.who.int/server/api/core/bitstreams/32214b73-0e95-4243-9e83-617948510dcd/content",
+            ),
+        ],
+    )
+
+
+def test_publication_ref_from_url_normalizes_publication_url() -> None:
+    """WHO publication URLs are normalized to canonical refs."""
+    assert publication_ref_from_url("https://www.who.int/publications/i/item/9789240121805") == WhoPublicationRef(
+        publication_id="9789240121805",
+        title="9789240121805",
+        page_url="https://www.who.int/publications/i/item/9789240121805",
+    )
+
+
+def test_build_publication_text_scrapes_overview() -> None:
+    """Publication pages are converted into Overview markdown."""
+    html = (_FIXTURES / "who_publication_overview.html").read_text(encoding="utf-8")
+    page_url = "https://www.who.int/publications/i/item/9789240121805"
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert str(request.url) == page_url
+        return httpx.Response(200, text=html)
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    ref = WhoPublicationRef(
+        publication_id="9789240121805",
+        title=_CVC_GUIDELINE_TITLE,
+        page_url=page_url,
+        publication_date="28 May 2026",
+        tag="Guideline",
+        download_url="https://iris.who.int/server/api/core/bitstreams/f750f24d-c0c2-425c-85fd-ec310d2ce994/content",
+    )
+
+    content, section_count, title, metadata = build_publication_text(client, ref)
+
+    assert title.startswith("Guidelines for the prevention of bloodstream infections")
+    assert section_count == 1
+    assert "### Overview" in content
+    assert "central venous catheters (CVCs)" in content
+    assert "WHO Team" not in content
+    assert metadata["publication_date"] == "28 May 2026"
+    assert metadata["tag"] == "Guideline"
+    assert metadata["isbn"] == "978-92-4-012180-5"
+    assert metadata["content_scope"] == "overview"
+    assert metadata["license"] == WHO_LICENSE
+    assert metadata["listing_category"] == "who-guidelines"
+
+
+def test_scrape_publication_builds_scraped_document() -> None:
+    """A WHO publication ref is normalized into a ScrapedDocument."""
+    html = (_FIXTURES / "who_publication_overview.html").read_text(encoding="utf-8")
+    page_url = "https://www.who.int/publications/i/item/9789240121805"
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(200, text=html)
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    ref = WhoPublicationRef(
+        publication_id="9789240121805",
+        title=_CVC_GUIDELINE_TITLE,
+        page_url=page_url,
+    )
+
+    document = scrape_publication(client, ref, link_mode=LinkMode.STRIP)
+
+    assert document.source == "who"
+    assert document.external_id == "who-9789240121805"
+    assert document.url == page_url
+    assert document.section_count == 1
+    assert document.content.strip()
+    assert document.metadata["publication_id"] == "9789240121805"
+    assert document.metadata["content_scope"] == "overview"
+
+
+def test_build_publication_text_raises_when_overview_missing() -> None:
+    """Missing publication markup raises WhoFetchError."""
+    html = "<html><body><p>No publication section here.</p></body></html>"
+    page_url = "https://www.who.int/publications/i/item/9789240121805"
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(200, text=html)
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    ref = WhoPublicationRef(
+        publication_id="9789240121805",
+        title="Example guideline",
+        page_url=page_url,
+    )
+
+    with pytest.raises(WhoFetchError, match="No publication content section"):
+        build_publication_text(client, ref)

From b6964d0391a702f2908fba06384a54dae031f23a Mon Sep 17 00:00:00 2001
From: Muhamed Kouate <muhamed.kouate@icloud.com>
Date: Wed, 8 Jul 2026 18:36:00 +0200
Subject: [PATCH 02/15] add IDSA scraper

---
 datasets/amfv_datasets/scraping/__init__.py |  28 ++
 datasets/amfv_datasets/scraping/cli.py      |  46 ++-
 datasets/amfv_datasets/scraping/idsa.py     | 399 ++++++++++++++++++++
 datasets/test/test_scraping_cli.py          | 132 +++++++
 datasets/test/test_scraping_idsa.py         | 320 ++++++++++++++++
 5 files changed, 921 insertions(+), 4 deletions(-)
 create mode 100644 datasets/amfv_datasets/scraping/idsa.py
 create mode 100644 datasets/test/test_scraping_idsa.py

diff --git a/datasets/amfv_datasets/scraping/__init__.py b/datasets/amfv_datasets/scraping/__init__.py
index cd7cdb06..d8a0aa01 100644
--- a/datasets/amfv_datasets/scraping/__init__.py
+++ b/datasets/amfv_datasets/scraping/__init__.py
@@ -17,6 +17,23 @@
     first_matching_urls,
     html_to_markdown,
 )
+from amfv_datasets.scraping.idsa import (
+    IDSA_DATASET_DISPLAY_NAME,
+    IDSA_DATASET_NAME,
+    LISTING_URL,
+    IDSAFetchError,
+    IDSAGuidelineListingPage,
+    IDSAGuidelineRef,
+    idsa_ref_from_url,
+    list_practice_guidelines,
+    scrape_idsa,
+)
+from amfv_datasets.scraping.idsa import (
+    build_guideline_text as build_idsa_guideline_text,
+)
+from amfv_datasets.scraping.idsa import (
+    scrape_guideline as scrape_idsa_guideline,
+)
 from amfv_datasets.scraping.nice import (
     GuidanceListingPage,
     GuidanceRef,
@@ -31,7 +48,13 @@
 __all__ = [
     "GuidanceRef",
     "GuidanceListingPage",
+    "IDSAFetchError",
+    "IDSA_DATASET_DISPLAY_NAME",
+    "IDSA_DATASET_NAME",
+    "IDSAGuidelineListingPage",
+    "IDSAGuidelineRef",
     "LinkMode",
+    "LISTING_URL",
     "NiceFetchError",
     "OutputFormat",
     "ScrapeError",
@@ -40,6 +63,7 @@
     "ScraperSource",
     "USER_AGENT",
     "absolute_unique_urls",
+    "build_idsa_guideline_text",
     "build_guideline_text",
     "clean_text",
     "document_title",
@@ -47,8 +71,12 @@
     "first_matching_urls",
     "guidance_ref_from_url",
     "html_to_markdown",
+    "idsa_ref_from_url",
+    "list_practice_guidelines",
     "list_published_guidance",
     "scrape_guideline",
+    "scrape_idsa",
+    "scrape_idsa_guideline",
     "scrape_listing_documents",
     "scrape_nice",
 ]
diff --git a/datasets/amfv_datasets/scraping/cli.py b/datasets/amfv_datasets/scraping/cli.py
index e8aece34..73170450 100644
--- a/datasets/amfv_datasets/scraping/cli.py
+++ b/datasets/amfv_datasets/scraping/cli.py
@@ -8,6 +8,7 @@
 from collections.abc import Iterable
 from dataclasses import asdict
 from enum import StrEnum
+from itertools import chain
 from pathlib import Path
 from typing import Annotated, TextIO
 
@@ -26,6 +27,7 @@
 
 from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun
 from amfv_datasets.scraping.html import LinkMode
+from amfv_datasets.scraping.idsa import scrape_idsa
 from amfv_datasets.scraping.nice import scrape_nice
 
 
@@ -33,6 +35,7 @@ class ScraperSource(StrEnum):
     """Supported scraper sources."""
 
     ALL = "all"
+    IDSA = "idsa"
     NICE = "nice"
 
 
@@ -53,6 +56,8 @@ def scrape_documents(
     documents: int | None,
     link_mode: LinkMode,
     url: str | None = None,
+    include_archived: bool = False,
+    include_in_development: bool = False,
 ) -> ScrapeRun:
     """Configure a scrape for a source.
 
@@ -64,17 +69,37 @@ def scrape_documents(
         link_mode: Whether links are kept as markdown links or stripped to their
             visible text.
         url: Source URL to scrape as a single document (default: None).
+        include_archived: Whether IDSA archived guidelines are included
+            (default: False).
+        include_in_development: Whether IDSA in-development guidelines are
+            included (default: False).
     """
     if documents is not None and documents < 1:
         raise ValueError(f"documents must be at least 1; got {documents}")
 
+    scrape_runs: list[ScrapeRun] = []
     for selected_source in _expand_source(source):
         match selected_source:
+            case ScraperSource.IDSA:
+                scrape_runs.append(
+                    scrape_idsa(
+                        documents=documents,
+                        link_mode=link_mode,
+                        url=url,
+                        include_archived=include_archived,
+                        include_in_development=include_in_development,
+                    )
+                )
             case ScraperSource.NICE:
-                return scrape_nice(documents=documents, link_mode=link_mode, url=url)
+                scrape_runs.append(scrape_nice(documents=documents, link_mode=link_mode, url=url))
             case ScraperSource.ALL:
                 raise AssertionError("expanded source cannot be all")
-    raise AssertionError(f"unsupported source: {source}")
+    if not scrape_runs:
+        raise AssertionError(f"unsupported source: {source}")
+    if len(scrape_runs) == 1:
+        return scrape_runs[0]
+    total = None if any(run.total is None for run in scrape_runs) else sum(run.total or 0 for run in scrape_runs)
+    return ScrapeRun(documents=chain.from_iterable(run.documents for run in scrape_runs), total=total)
 
 
 def write_jsonl(documents: Iterable[ScrapedDocument], output: TextIO) -> int:
@@ -125,7 +150,7 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path
 
 def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]:
     if source is ScraperSource.ALL:
-        return (ScraperSource.NICE,)
+        return (ScraperSource.NICE, ScraperSource.IDSA)
     return (source,)
 
 
@@ -135,6 +160,8 @@ def run(
     url: Annotated[str | None, typer.Option("--url", help="Source URL to scrape as a single document.")] = None,
     documents: Annotated[str, typer.Option("--documents", help="Number of documents to scrape, or 'all'.")] = "1",
     link_mode: Annotated[LinkMode, typer.Option("--links", help="Whether to keep markdown links or strip links to text.")] = LinkMode.KEEP,  # noqa: E501
+    include_archived: Annotated[bool, typer.Option("--include-archived/--exclude-archived", help="Include archived guidelines for sources that expose archival status.")] = False,  # noqa: E501
+    include_in_development: Annotated[bool, typer.Option("--include-in-development/--exclude-in-development", help="Include in-development guidelines for sources that expose development status.")] = False,  # noqa: E501
     output_format: Annotated[OutputFormat, typer.Option("--format", "-f", help="Output format.")] = OutputFormat.JSONL,
     output_path: Annotated[Path | None, typer.Option("--output", "-o", help="Output JSONL file, markdown directory, or Hugging Face dataset directory. JSONL defaults to stdout.")] = None,  # noqa: E501
     progress: Annotated[bool, typer.Option("--progress/--no-progress", help="Show a Rich progress bar.")] = True,
@@ -147,6 +174,10 @@ def run(
         documents: Number of documents to scrape, or "all" (default: "1").
         link_mode: Whether links are kept as markdown links or stripped to their
             visible text (default: LinkMode.KEEP).
+        include_archived: Whether archived guidelines are included for sources
+            that expose archival status (default: False).
+        include_in_development: Whether in-development guidelines are included
+            for sources that expose development status (default: False).
         output_format: Output format to write (default: OutputFormat.JSONL).
         output_path: Output JSONL file, markdown directory, or Hugging Face
             dataset directory. When unset, JSONL is written to stdout (default:
@@ -154,7 +185,14 @@ def run(
         progress: Whether to show a Rich progress bar (default: True).
     """
     parsed_documents = _parse_documents(documents)
-    scrape_run = scrape_documents(source, documents=parsed_documents, link_mode=link_mode, url=url)
+    scrape_run = scrape_documents(
+        source,
+        documents=parsed_documents,
+        link_mode=link_mode,
+        url=url,
+        include_archived=include_archived,
+        include_in_development=include_in_development,
+    )
     scraped_documents = scrape_run.documents
     if progress:
         scraped_documents = _progress_documents(scraped_documents, total=scrape_run.total)
diff --git a/datasets/amfv_datasets/scraping/idsa.py b/datasets/amfv_datasets/scraping/idsa.py
new file mode 100644
index 00000000..9941b3f1
--- /dev/null
+++ b/datasets/amfv_datasets/scraping/idsa.py
@@ -0,0 +1,399 @@
+"""Scrape IDSA practice guidelines into normalized markdown documents."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from dataclasses import dataclass
+from urllib.parse import urljoin, urlparse
+
+import httpx
+from lxml import html as lxml_html
+
+from amfv_datasets.scraping.base import (
+    ScrapedDocument,
+    ScrapeError,
+    ScrapeRun,
+    default_client,
+    scrape_listing_documents,
+)
+from amfv_datasets.scraping.html import LinkMode, absolute_unique_urls, clean_text, document_title, html_to_markdown
+
+BASE_URL = "https://www.idsociety.org"
+LISTING_URL = f"{BASE_URL}/practice-guideline/all-practice-guidelines"
+IDSA_DATASET_NAME = "idsa-webscrape"
+IDSA_DATASET_DISPLAY_NAME = "IDSA Webscrape"
+DOCUMENT_DELAY_SECONDS = 5.0
+SHORT_CONTENT_CHARS = 10_000
+
+_IDSA_PATH_RE = re.compile(r"^/practice-guideline/(?P<slug>[^/]+)(?:/|$)", re.IGNORECASE)
+_BACK_TO_TOP_RE = re.compile(r"^\s*back to top\s*$", re.IGNORECASE)
+_TABLE_OF_CONTENTS_RE = re.compile(r"^\s*table\s+of\s+contents\s*$", re.IGNORECASE)
+_PDF_TEXT_RE = re.compile(r"\b(download\s+)?pdf\b", re.IGNORECASE)
+_WHITESPACE_RE = re.compile(r"\s+")
+
+
+class IDSAFetchError(ScrapeError):
+    """Raised when a practice guideline cannot be sourced from IDSA."""
+
+
+@dataclass(frozen=True)
+class IDSAGuidelineRef:
+    """An IDSA practice guideline reference from the A-Z listing."""
+
+    title: str
+    slug: str
+    page_url: str
+    year: int | None
+    statuses: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class IDSAGuidelineListingPage:
+    """An IDSA practice guideline listing page."""
+
+    refs: list[IDSAGuidelineRef]
+    total: int | None
+
+
+def idsa_ref_from_url(url: str) -> IDSAGuidelineRef:
+    """Parse an IDSA practice guideline URL into a canonical guideline reference.
+
+    Args:
+        url: IDSA practice guideline URL to parse.
+    """
+    parsed = urlparse(url.strip())
+    if parsed.scheme not in {"http", "https"} or parsed.netloc.lower() not in {
+        "www.idsociety.org",
+        "idsociety.org",
+    }:
+        raise IDSAFetchError(f"Enter an IDSA practice guideline URL from idsociety.org; got {url!r}")
+
+    match = _IDSA_PATH_RE.match(parsed.path.rstrip("/") + "/")
+    if not match:
+        raise IDSAFetchError(f"Enter a URL like https://www.idsociety.org/practice-guideline/example/; got {url!r}")
+
+    slug = match.group("slug").lower()
+    return IDSAGuidelineRef(
+        title=slug.replace("-", " ").title(),
+        slug=slug,
+        page_url=_page_url(slug),
+        year=None,
+        statuses=(),
+    )
+
+
+def list_practice_guidelines(
+    client: httpx.Client,
+    *,
+    include_archived: bool = False,
+    include_in_development: bool = False,
+) -> IDSAGuidelineListingPage:
+    """Return IDSA practice guideline refs from the A-Z listing.
+
+    Args:
+        client: HTTP client used to fetch the listing page.
+        include_archived: Whether archived guidelines are included (default: False).
+        include_in_development: Whether in-development guidelines are included (default: False).
+    """
+    response = client.get(LISTING_URL)
+    response.raise_for_status()
+    refs = _parse_listing(
+        response.text,
+        include_archived=include_archived,
+        include_in_development=include_in_development,
+    )
+    return IDSAGuidelineListingPage(refs=refs, total=len(refs))
+
+
+def build_guideline_text(
+    client: httpx.Client,
+    ref: IDSAGuidelineRef,
+    *,
+    link_mode: LinkMode = LinkMode.KEEP,
+) -> tuple[str, int, str, dict[str, list[str]]]:
+    """Scrape a guideline page into markdown text, section count, title, and links.
+
+    Args:
+        client: HTTP client used to fetch the guideline page.
+        ref: IDSA guideline reference to scrape.
+        link_mode: Whether links are kept as markdown links or stripped to their
+            visible text (default: LinkMode.KEEP).
+    """
+    response = client.get(ref.page_url)
+    response.raise_for_status()
+    title = document_title(response.text, fallback=ref.title)
+    content_html = _guideline_content_html(response.text)
+    content = html_to_markdown(content_html, link_mode=link_mode, base_url=BASE_URL)
+    if not content:
+        raise IDSAFetchError(f"No readable content for IDSA guideline '{ref.slug}'")
+    return content, _section_count(content_html), title, _links_metadata(content_html)
+
+
+def scrape_guideline(
+    client: httpx.Client,
+    ref: IDSAGuidelineRef,
+    *,
+    link_mode: LinkMode = LinkMode.KEEP,
+) -> ScrapedDocument:
+    """Scrape an IDSA practice guideline into a normalized document.
+
+    Args:
+        client: HTTP client used to fetch the guideline page.
+        ref: IDSA guideline reference to scrape.
+        link_mode: Whether links are kept as markdown links or stripped to their
+            visible text (default: LinkMode.KEEP).
+    """
+    content, section_count, title, links_metadata = build_guideline_text(client, ref, link_mode=link_mode)
+    return ScrapedDocument(
+        source="idsa",
+        external_id=f"idsa-{ref.slug}",
+        title=title,
+        url=ref.page_url,
+        content=content,
+        section_count=section_count,
+        metadata={
+            "year": ref.year,
+            "statuses": list(ref.statuses),
+            "slug": ref.slug,
+            "listing_url": LISTING_URL,
+            "content_length_chars": len(content),
+            "quality_flags": _quality_flags(content),
+            **links_metadata,
+        },
+    )
+
+
+def scrape_idsa(
+    *,
+    documents: int | None,
+    link_mode: LinkMode = LinkMode.KEEP,
+    url: str | None = None,
+    include_archived: bool = False,
+    include_in_development: bool = False,
+) -> ScrapeRun:
+    """Scrape IDSA practice guidelines from a URL or the A-Z listing.
+
+    Args:
+        documents: Number of documents to scrape. Ignored when `url` is set.
+            When unset, every included IDSA listing item is scraped (default:
+            None).
+        link_mode: Whether links are kept as markdown links or stripped to their
+            visible text (default: LinkMode.KEEP).
+        url: IDSA source URL to scrape as a single document (default: None).
+        include_archived: Whether archived guidelines are included (default: False).
+        include_in_development: Whether in-development guidelines are included
+            (default: False).
+    """
+    if url is not None:
+
+        def scrape_url() -> Iterable[ScrapedDocument]:
+            with default_client() as client:
+                yield scrape_guideline(client, idsa_ref_from_url(url), link_mode=link_mode)
+
+        return ScrapeRun(documents=scrape_url(), total=1)
+
+    with default_client() as client:
+        listing = list_practice_guidelines(
+            client,
+            include_archived=include_archived,
+            include_in_development=include_in_development,
+        )
+    total = listing.total if documents is None or listing.total is None else min(documents, listing.total)
+    return ScrapeRun(
+        total=total,
+        documents=scrape_listing_documents(
+            documents=documents,
+            client_factory=default_client,
+            first_page_items=listing.refs,
+            list_page=lambda _client, _page: (),
+            scrape_item=lambda client, ref: scrape_guideline(client, ref, link_mode=link_mode),
+            document_delay_seconds=DOCUMENT_DELAY_SECONDS,
+        ),
+    )
+
+
+def _parse_listing(
+    html_text: str,
+    *,
+    include_archived: bool,
+    include_in_development: bool,
+) -> list[IDSAGuidelineRef]:
+    doc = lxml_html.fromstring(html_text)
+    items = doc.xpath(
+        "//div[contains(concat(' ', normalize-space(@class), ' '), ' alpha-listing ')]"
+        "//li[.//a[contains(concat(' ', normalize-space(@class), ' '), ' list-pages__link ')]]"
+    )
+    refs: list[IDSAGuidelineRef] = []
+    for item in items:
+        link = item.xpath(".//a[contains(concat(' ', normalize-space(@class), ' '), ' list-pages__link ')][1]")
+        if not link:
+            continue
+        statuses = tuple(
+            clean_text(status, drop_numeric_citations=False)
+            for status in item.xpath(
+                ".//*[contains(concat(' ', normalize-space(@class), ' '), ' category-dot ')]/text()"
+            )
+        )
+        if not _included_statuses(
+            statuses,
+            include_archived=include_archived,
+            include_in_development=include_in_development,
+        ):
+            continue
+        href = link[0].get("href")
+        if not href:
+            continue
+        page_url = urljoin(BASE_URL, href).split("#")[0].split("?")[0]
+        match = _IDSA_PATH_RE.match(urlparse(page_url).path.rstrip("/") + "/")
+        if not match:
+            continue
+        refs.append(
+            IDSAGuidelineRef(
+                title=clean_text(link[0].text_content(), drop_numeric_citations=False),
+                slug=match.group("slug").lower(),
+                page_url=page_url,
+                year=_parse_year(item),
+                statuses=statuses,
+            )
+        )
+    return refs
+
+
+def _included_statuses(
+    statuses: tuple[str, ...],
+    *,
+    include_archived: bool,
+    include_in_development: bool,
+) -> bool:
+    has_current = "Current" in statuses
+    has_archived = "Archived" in statuses
+    has_in_development = "In Development" in statuses
+    if has_archived and not include_archived:
+        return False
+    if has_in_development and not include_in_development:
+        return False
+    return has_current or (has_archived and include_archived) or (has_in_development and include_in_development)
+
+
+def _parse_year(item: lxml_html.HtmlElement) -> int | None:
+    values = item.xpath(".//*[contains(concat(' ', normalize-space(@class), ' '), ' list-pages__year ')]/text()")
+    if not values:
+        return None
+    value = clean_text(values[0], drop_numeric_citations=False)
+    return int(value) if value.isdigit() else None
+
+
+def _guideline_content_html(html_text: str) -> str:
+    doc = lxml_html.fromstring(html_text)
+    candidates = doc.xpath("//div[contains(concat(' ', normalize-space(@class), ' '), ' idsaPracticeGuidelinePage ')]")
+    if not candidates:
+        candidates = doc.xpath("//*[contains(concat(' ', normalize-space(@class), ' '), ' body-container ')]")
+    if not candidates:
+        candidates = doc.xpath("//div[contains(concat(' ', normalize-space(@class), ' '), ' standardpage-col-left ')]")
+    if not candidates:
+        raise IDSAFetchError("Page markup changed (no IDSA guideline content container)")
+    content = candidates[-1]
+    _remove_noise(content)
+    content_html = lxml_html.tostring(content, encoding="unicode")
+    if not clean_text(content.text_content(), drop_numeric_citations=False):
+        raise IDSAFetchError("Page markup changed (empty IDSA guideline content container)")
+    return content_html
+
+
+def _remove_noise(content: lxml_html.HtmlElement) -> None:
+    noise_xpath = (
+        ".//*[self::script or self::style or self::noscript or self::svg or self::button"
+        " or contains(concat(' ', normalize-space(@class), ' '), ' table-of-contents ')"
+        " or contains(concat(' ', normalize-space(@class), ' '), ' toc ')"
+        " or contains(concat(' ', normalize-space(@class), ' '), ' TableOfContents ')"
+        " or contains(concat(' ', normalize-space(@class), ' '), ' status-section ')"
+        " or contains(concat(' ', normalize-space(@class), ' '), ' view-all-guidance ')"
+        " or contains(concat(' ', normalize-space(@class), ' '), ' share ')"
+        " or contains(concat(' ', normalize-space(@class), ' '), ' addthis ')"
+        " or contains(concat(' ', normalize-space(@class), ' '), ' social ')]"
+    )
+    for element in content.xpath(noise_xpath):
+        _drop_element(element)
+    _remove_table_of_contents(content)
+    for element in content.xpath(".//*[self::a or self::p or self::div or self::span]"):
+        if _BACK_TO_TOP_RE.match(element.text_content()):
+            _drop_element(element)
+
+
+def _remove_table_of_contents(content: lxml_html.HtmlElement) -> None:
+    headings = content.xpath(".//*[self::h1 or self::h2 or self::h3 or self::h4 or self::h5 or self::h6]")
+    for heading in headings:
+        if not _TABLE_OF_CONTENTS_RE.match(heading.text_content()):
+            continue
+        for sibling in list(heading.itersiblings()):
+            if _heading_level(sibling) is not None:
+                break
+            _drop_element(sibling)
+        _drop_element(heading)
+
+
+def _heading_level(element: lxml_html.HtmlElement) -> int | None:
+    tag = element.tag.lower() if isinstance(element.tag, str) else ""
+    if len(tag) == 2 and tag.startswith("h") and tag[1].isdigit():
+        return int(tag[1])
+    return None
+
+
+def _drop_element(element: lxml_html.HtmlElement) -> None:
+    parent = element.getparent()
+    if parent is not None:
+        parent.remove(element)
+
+
+def _links_metadata(content_html: str) -> dict[str, list[str]]:
+    doc = lxml_html.fromstring(content_html)
+    external_links: list[str] = []
+    pdf_links: list[str] = []
+    for link in doc.xpath(".//a[@href]"):
+        url = urljoin(BASE_URL, link.get("href")).split("#")[0].split("?")[0]
+        if urlparse(url).netloc.lower() in {"www.idsociety.org", "idsociety.org"}:
+            continue
+        external_links.append(url)
+        link_text = _WHITESPACE_RE.sub(" ", link.text_content()).strip()
+        if url.lower().endswith(".pdf") or _PDF_TEXT_RE.search(link_text):
+            pdf_links.append(url)
+    return {
+        "external_links": absolute_unique_urls(external_links, base_url=BASE_URL),
+        "pdf_links": absolute_unique_urls(pdf_links, base_url=BASE_URL),
+    }
+
+
+def _section_count(content_html: str) -> int:
+    doc = lxml_html.fromstring(content_html)
+    headings = doc.xpath(".//*[self::h1 or self::h2 or self::h3]")
+    return max(1, len(headings))
+
+
+def _quality_flags(content: str) -> list[str]:
+    flags: list[str] = []
+    if len(content) < SHORT_CONTENT_CHARS:
+        flags.append("short_content")
+    return flags
+
+
+def _page_url(slug: str) -> str:
+    return f"{BASE_URL}/practice-guideline/{slug}/"
+
+
+__all__ = [
+    "BASE_URL",
+    "DOCUMENT_DELAY_SECONDS",
+    "IDSA_DATASET_DISPLAY_NAME",
+    "IDSA_DATASET_NAME",
+    "IDSAFetchError",
+    "IDSAGuidelineListingPage",
+    "IDSAGuidelineRef",
+    "LISTING_URL",
+    "SHORT_CONTENT_CHARS",
+    "build_guideline_text",
+    "idsa_ref_from_url",
+    "list_practice_guidelines",
+    "scrape_guideline",
+    "scrape_idsa",
+]
diff --git a/datasets/test/test_scraping_cli.py b/datasets/test/test_scraping_cli.py
index 44f714fa..bc4eccd2 100644
--- a/datasets/test/test_scraping_cli.py
+++ b/datasets/test/test_scraping_cli.py
@@ -70,11 +70,15 @@ def fake_scrape_documents(
         documents: int | None,
         link_mode: LinkMode,
         url: str | None = None,
+        include_archived: bool = False,
+        include_in_development: bool = False,
     ) -> ScrapeRun:
         assert source is ScraperSource.NICE
         assert documents == 3
         assert link_mode is LinkMode.STRIP
         assert url is None
+        assert include_archived is False
+        assert include_in_development is False
         return ScrapeRun([_document()], total=3)
 
     monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_documents", fake_scrape_documents)
@@ -105,6 +109,8 @@ def fake_scrape_documents(
         documents: int | None,
         link_mode: LinkMode,
         url: str | None = None,
+        include_archived: bool = False,
+        include_in_development: bool = False,
     ) -> ScrapeRun:
         return ScrapeRun([_document()], total=None)
 
@@ -137,6 +143,8 @@ def fake_scrape_documents(
         documents: int | None,
         link_mode: LinkMode,
         url: str | None = None,
+        include_archived: bool = False,
+        include_in_development: bool = False,
     ) -> ScrapeRun:
         return ScrapeRun([_document()], total=7)
 
@@ -204,11 +212,15 @@ def fake_scrape_documents(
         documents: int | None,
         link_mode: LinkMode,
         url: str | None = None,
+        include_archived: bool = False,
+        include_in_development: bool = False,
     ) -> ScrapeRun:
         assert source is ScraperSource.ALL
         assert documents is None
         assert link_mode is LinkMode.KEEP
         assert url is None
+        assert include_archived is False
+        assert include_in_development is False
         return ScrapeRun([_document()], total=12)
 
     monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_documents", fake_scrape_documents)
@@ -219,6 +231,115 @@ def fake_scrape_documents(
     assert json.loads(result.stdout.splitlines()[0])["external_id"] == "nice-ng1"
 
 
+def test_cli_run_accepts_idsa_status_flags(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The CLI forwards IDSA status inclusion flags."""
+    runner = CliRunner()
+
+    def fake_scrape_documents(
+        source: ScraperSource,
+        *,
+        documents: int | None,
+        link_mode: LinkMode,
+        url: str | None = None,
+        include_archived: bool = False,
+        include_in_development: bool = False,
+    ) -> ScrapeRun:
+        assert source is ScraperSource.IDSA
+        assert documents == 2
+        assert link_mode is LinkMode.KEEP
+        assert url is None
+        assert include_archived is True
+        assert include_in_development is True
+        return ScrapeRun([_idsa_document()], total=2)
+
+    monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_documents", fake_scrape_documents)
+
+    result = runner.invoke(
+        app,
+        ["--source", "idsa", "--documents", "2", "--include-archived", "--include-in-development"],
+    )
+
+    assert result.exit_code == 0
+    assert json.loads(result.stdout.splitlines()[0])["external_id"] == "idsa-current-guideline"
+    assert "scraped 1 documents from idsa" in result.stderr
+
+
+def test_scrape_documents_dispatches_idsa(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The scrape dispatcher calls the IDSA scraper for the IDSA source."""
+
+    def fake_scrape_idsa(
+        *,
+        documents: int | None,
+        link_mode: LinkMode,
+        url: str | None,
+        include_archived: bool,
+        include_in_development: bool,
+    ) -> ScrapeRun:
+        assert documents == 1
+        assert link_mode is LinkMode.STRIP
+        assert url == "https://www.idsociety.org/practice-guideline/current-guideline/"
+        assert include_archived is True
+        assert include_in_development is False
+        return ScrapeRun([_idsa_document()], total=1)
+
+    monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_idsa", fake_scrape_idsa)
+
+    from amfv_datasets.scraping.cli import scrape_documents
+
+    scrape_run = scrape_documents(
+        ScraperSource.IDSA,
+        documents=1,
+        link_mode=LinkMode.STRIP,
+        url="https://www.idsociety.org/practice-guideline/current-guideline/",
+        include_archived=True,
+    )
+
+    assert list(scrape_run.documents) == [_idsa_document()]
+
+
+def test_scrape_documents_all_includes_idsa(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The all source combines NICE and IDSA scrape runs."""
+
+    def fake_scrape_nice(
+        *,
+        documents: int | None,
+        link_mode: LinkMode,
+        url: str | None,
+    ) -> ScrapeRun:
+        assert documents == 1
+        assert link_mode is LinkMode.KEEP
+        assert url is None
+        return ScrapeRun([_document()], total=1)
+
+    def fake_scrape_idsa(
+        *,
+        documents: int | None,
+        link_mode: LinkMode,
+        url: str | None,
+        include_archived: bool,
+        include_in_development: bool,
+    ) -> ScrapeRun:
+        assert documents == 1
+        assert link_mode is LinkMode.KEEP
+        assert url is None
+        assert include_archived is False
+        assert include_in_development is False
+        return ScrapeRun([_idsa_document()], total=1)
+
+    monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_nice", fake_scrape_nice)
+    monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_idsa", fake_scrape_idsa)
+
+    from amfv_datasets.scraping.cli import scrape_documents
+
+    scrape_run = scrape_documents(ScraperSource.ALL, documents=1, link_mode=LinkMode.KEEP)
+
+    assert scrape_run.total == 2
+    assert [document.external_id for document in scrape_run.documents] == [
+        "nice-ng1",
+        "idsa-current-guideline",
+    ]
+
+
 def _document() -> ScrapedDocument:
     return ScrapedDocument(
         source="nice",
@@ -230,6 +351,17 @@ def _document() -> ScrapedDocument:
     )
 
 
+def _idsa_document() -> ScrapedDocument:
+    return ScrapedDocument(
+        source="idsa",
+        external_id="idsa-current-guideline",
+        title="Current Guideline",
+        url="https://www.idsociety.org/practice-guideline/current-guideline/",
+        content="content",
+        metadata={"slug": "current-guideline"},
+    )
+
+
 class _TextSink:
     def __init__(self) -> None:
         self.value = ""
diff --git a/datasets/test/test_scraping_idsa.py b/datasets/test/test_scraping_idsa.py
new file mode 100644
index 00000000..a83c4ae9
--- /dev/null
+++ b/datasets/test/test_scraping_idsa.py
@@ -0,0 +1,320 @@
+"""Tests for IDSA scraping helpers."""
+
+import httpx
+import pytest
+
+from amfv_datasets.scraping.html import LinkMode
+from amfv_datasets.scraping.idsa import (
+    BASE_URL,
+    LISTING_URL,
+    IDSAFetchError,
+    IDSAGuidelineRef,
+    build_guideline_text,
+    idsa_ref_from_url,
+    list_practice_guidelines,
+    scrape_guideline,
+    scrape_idsa,
+)
+
+
+def test_list_practice_guidelines_parses_and_filters_default_statuses() -> None:
+    """The IDSA listing includes only current non-development guidelines by default."""
+    client = httpx.Client(transport=httpx.MockTransport(_listing_handler), base_url=BASE_URL)
+
+    listing = list_practice_guidelines(client)
+
+    assert listing.total == 2
+    assert listing.refs == [
+        IDSAGuidelineRef(
+            title="Current Guideline",
+            slug="current-guideline",
+            page_url="https://www.idsociety.org/practice-guideline/current-guideline/",
+            year=2024,
+            statuses=("Current",),
+        ),
+        IDSAGuidelineRef(
+            title="Current Endorsed Guideline",
+            slug="current-endorsed-guideline",
+            page_url="https://www.idsociety.org/practice-guideline/current-endorsed-guideline/",
+            year=2023,
+            statuses=("Current", "Endorsed"),
+        ),
+    ]
+
+
+def test_list_practice_guidelines_can_include_archived_statuses() -> None:
+    """Archived guidelines are included only when requested."""
+    client = httpx.Client(transport=httpx.MockTransport(_listing_handler), base_url=BASE_URL)
+
+    listing = list_practice_guidelines(client, include_archived=True)
+
+    assert [ref.slug for ref in listing.refs] == [
+        "current-guideline",
+        "current-endorsed-guideline",
+        "archived-guideline",
+    ]
+
+
+def test_list_practice_guidelines_can_include_in_development_statuses() -> None:
+    """In-development guidelines are included only when requested."""
+    client = httpx.Client(transport=httpx.MockTransport(_listing_handler), base_url=BASE_URL)
+
+    listing = list_practice_guidelines(client, include_in_development=True)
+
+    assert [ref.slug for ref in listing.refs] == [
+        "current-guideline",
+        "current-endorsed-guideline",
+        "development-guideline",
+    ]
+
+
+def test_list_practice_guidelines_requires_both_flags_for_archived_development_statuses() -> None:
+    """Guidelines marked both archived and in development require both inclusion flags."""
+    client = httpx.Client(transport=httpx.MockTransport(_listing_handler), base_url=BASE_URL)
+
+    listing = list_practice_guidelines(client, include_archived=True, include_in_development=True)
+
+    assert [ref.slug for ref in listing.refs] == [
+        "current-guideline",
+        "current-endorsed-guideline",
+        "archived-guideline",
+        "development-guideline",
+        "archived-development-guideline",
+    ]
+
+
+def test_idsa_ref_from_url_normalizes_practice_guideline_url() -> None:
+    """IDSA practice guideline URLs are normalized to canonical refs."""
+    assert idsa_ref_from_url("https://www.idsociety.org/practice-guideline/Current-Guideline/?utm=1") == (
+        IDSAGuidelineRef(
+            title="Current Guideline",
+            slug="current-guideline",
+            page_url="https://www.idsociety.org/practice-guideline/current-guideline/",
+            year=None,
+            statuses=(),
+        )
+    )
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "https://example.com/practice-guideline/current-guideline/",
+        "https://www.idsociety.org/news/current-guideline/",
+    ],
+    ids=["wrong-domain", "wrong-path"],
+)
+def test_idsa_ref_from_url_rejects_non_guideline_urls(url: str) -> None:
+    """Only IDSA practice guideline URLs are accepted."""
+    with pytest.raises(IDSAFetchError):
+        idsa_ref_from_url(url)
+
+
+def test_build_guideline_text_extracts_content_and_link_metadata() -> None:
+    """Guideline page content is converted to markdown and noisy UI is stripped."""
+    client = httpx.Client(transport=httpx.MockTransport(_guideline_handler), base_url=BASE_URL)
+
+    content, section_count, title, links_metadata = build_guideline_text(
+        client,
+        IDSAGuidelineRef(
+            title="Current Guideline",
+            slug="current-guideline",
+            page_url="https://www.idsociety.org/practice-guideline/current-guideline/",
+            year=2024,
+            statuses=("Current",),
+        ),
+    )
+
+    assert title == "Current Guideline"
+    assert section_count == 3
+    assert "# Current Guideline" in content
+    assert "## Abstract" in content
+    assert "## Recommendations" in content
+    assert "Recommendation text with [evidence](https://doi.org/10.1093/cid/example)." in content
+    assert "Back to top" not in content
+    assert "Table of Contents" not in content
+    assert "https://www.idsociety.org#abstract" not in content
+    assert links_metadata == {
+        "external_links": [
+            "https://academic.oup.com/example.pdf",
+            "https://doi.org/10.1093/cid/example",
+        ],
+        "pdf_links": ["https://academic.oup.com/example.pdf"],
+    }
+
+    stripped_content, _section_count, _title, _links_metadata = build_guideline_text(
+        client,
+        IDSAGuidelineRef(
+            title="Current Guideline",
+            slug="current-guideline",
+            page_url="https://www.idsociety.org/practice-guideline/current-guideline/",
+            year=2024,
+            statuses=("Current",),
+        ),
+        link_mode=LinkMode.STRIP,
+    )
+    assert "Recommendation text with evidence." in stripped_content
+
+
+def test_scrape_idsa_returns_normalized_document() -> None:
+    """The IDSA scraper returns normalized scraped documents."""
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        if str(request.url) == LISTING_URL:
+            return httpx.Response(200, text=_listing_html())
+        if str(request.url) == "https://www.idsociety.org/practice-guideline/current-guideline/":
+            return httpx.Response(200, text=_guideline_html())
+        raise AssertionError(f"Unexpected URL: {request.url}")
+
+    original_client_factory = "amfv_datasets.scraping.idsa.default_client"
+    transport = httpx.MockTransport(handler)
+
+    class ClientFactory:
+        def __call__(self) -> httpx.Client:
+            return httpx.Client(transport=transport)
+
+    with pytest.MonkeyPatch.context() as monkeypatch:
+        monkeypatch.setattr(original_client_factory, ClientFactory())
+        scrape_run = scrape_idsa(documents=1)
+        documents = list(scrape_run.documents)
+
+    assert scrape_run.total == 1
+    assert len(documents) == 1
+    assert documents[0].source == "idsa"
+    assert documents[0].external_id == "idsa-current-guideline"
+    assert documents[0].metadata["statuses"] == ["Current"]
+    assert documents[0].metadata["year"] == 2024
+    assert documents[0].metadata["pdf_links"] == ["https://academic.oup.com/example.pdf"]
+    assert documents[0].metadata["content_length_chars"] == len(documents[0].content)
+    assert documents[0].metadata["quality_flags"] == ["short_content"]
+
+
+def test_scrape_guideline_does_not_flag_long_content() -> None:
+    """Long guideline content records length without short-content quality flags."""
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert str(request.url) == "https://www.idsociety.org/practice-guideline/long-guideline/"
+        return httpx.Response(200, text=_long_guideline_html())
+
+    client = httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL)
+
+    document = scrape_guideline(
+        client,
+        IDSAGuidelineRef(
+            title="Long Guideline",
+            slug="long-guideline",
+            page_url="https://www.idsociety.org/practice-guideline/long-guideline/",
+            year=2024,
+            statuses=("Current",),
+        ),
+    )
+
+    assert document.metadata["content_length_chars"] == len(document.content)
+    assert document.metadata["quality_flags"] == []
+
+
+def _listing_handler(request: httpx.Request) -> httpx.Response:
+    assert str(request.url) == LISTING_URL
+    return httpx.Response(200, text=_listing_html())
+
+
+def _guideline_handler(request: httpx.Request) -> httpx.Response:
+    assert str(request.url) == "https://www.idsociety.org/practice-guideline/current-guideline/"
+    return httpx.Response(200, text=_guideline_html())
+
+
+def _listing_html() -> str:
+    return """
+        <html>
+          <div class="alpha-listing">
+            <ul class="list-pages">
+              <li>
+                <ul class="list-pages__categories"><li class="category-dot category-dot-current">Current</li></ul>
+                <a class="list-pages__link" href="/practice-guideline/current-guideline/">Current Guideline</a>
+                <span class="list-pages__year">2024</span>
+              </li>
+              <li>
+                <ul class="list-pages__categories">
+                  <li class="category-dot category-dot-current">Current</li>
+                  <li class="category-dot category-dot-endorsed">Endorsed</li>
+                </ul>
+                <a class="list-pages__link" href="/practice-guideline/current-endorsed-guideline/">
+                  Current Endorsed Guideline
+                </a>
+                <span class="list-pages__year">2023</span>
+              </li>
+              <li>
+                <ul class="list-pages__categories"><li class="category-dot category-dot-endorsed">Endorsed</li></ul>
+                <a class="list-pages__link" href="/practice-guideline/endorsed-guideline/">Endorsed Guideline</a>
+                <span class="list-pages__year">2022</span>
+              </li>
+              <li>
+                <ul class="list-pages__categories"><li class="category-dot category-dot-archived">Archived</li></ul>
+                <a class="list-pages__link" href="/practice-guideline/archived-guideline/">Archived Guideline</a>
+                <span class="list-pages__year">2021</span>
+              </li>
+              <li>
+                <ul class="list-pages__categories">
+                  <li class="category-dot category-dot-in-development">In Development</li>
+                </ul>
+                <a class="list-pages__link" href="/practice-guideline/development-guideline/">
+                  Development Guideline
+                </a>
+                <span class="list-pages__year">2020</span>
+              </li>
+              <li>
+                <ul class="list-pages__categories">
+                  <li class="category-dot category-dot-archived">Archived</li>
+                  <li class="category-dot category-dot-in-development">In Development</li>
+                </ul>
+                <a class="list-pages__link" href="/practice-guideline/archived-development-guideline/">
+                  Archived Development Guideline
+                </a>
+                <span class="list-pages__year">2019</span>
+              </li>
+            </ul>
+          </div>
+        </html>
+    """
+
+
+def _guideline_html() -> str:
+    return """
+        <html>
+          <title>Current Guideline | IDSA</title>
+          <div class="standardpage-col-left">
+            <nav>Navigation noise</nav>
+            <p>Intro column noise.</p>
+          </div>
+          <div class="standardpage-col-left">
+            <p><a href="https://academic.oup.com/example.pdf">Download PDF</a></p>
+            <h1>Current Guideline</h1>
+            <h3>Table of Contents</h3>
+            <ul>
+              <li><a href="#abstract">Abstract</a></li>
+              <li><a href="#recommendations">Recommendations</a></li>
+            </ul>
+            <p class="social">Share this page</p>
+            <p>Published January 1, 2024</p>
+            <h2>Abstract</h2>
+            <p>Useful abstract.</p>
+            <h2>Recommendations</h2>
+            <p>Recommendation text with <a href="https://doi.org/10.1093/cid/example">evidence</a>.</p>
+            <p>Back to top</p>
+          </div>
+        </html>
+    """
+
+
+def _long_guideline_html() -> str:
+    long_text = "Recommendation text. " * 700
+    return f"""
+        <html>
+          <title>Long Guideline | IDSA</title>
+          <div class="standardpage-col-left">
+            <h1>Long Guideline</h1>
+            <h2>Recommendations</h2>
+            <p>{long_text}</p>
+          </div>
+        </html>
+    """

From 5a157e47eaa6c4dcf169404531bfc03bccd20d6f Mon Sep 17 00:00:00 2001
From: Muhamed Kouate <muhamed.kouate@icloud.com>
Date: Wed, 8 Jul 2026 19:04:51 +0200
Subject: [PATCH 03/15] simplify IDSA scraper

---
 datasets/amfv_datasets/scraping/__init__.py |  12 -
 datasets/amfv_datasets/scraping/idsa.py     |  22 +-
 datasets/test/test_scraping_idsa.py         | 318 ++++++++------------
 3 files changed, 132 insertions(+), 220 deletions(-)

diff --git a/datasets/amfv_datasets/scraping/__init__.py b/datasets/amfv_datasets/scraping/__init__.py
index d8a0aa01..199e053a 100644
--- a/datasets/amfv_datasets/scraping/__init__.py
+++ b/datasets/amfv_datasets/scraping/__init__.py
@@ -18,19 +18,12 @@
     html_to_markdown,
 )
 from amfv_datasets.scraping.idsa import (
-    IDSA_DATASET_DISPLAY_NAME,
-    IDSA_DATASET_NAME,
-    LISTING_URL,
     IDSAFetchError,
-    IDSAGuidelineListingPage,
     IDSAGuidelineRef,
     idsa_ref_from_url,
     list_practice_guidelines,
     scrape_idsa,
 )
-from amfv_datasets.scraping.idsa import (
-    build_guideline_text as build_idsa_guideline_text,
-)
 from amfv_datasets.scraping.idsa import (
     scrape_guideline as scrape_idsa_guideline,
 )
@@ -49,12 +42,8 @@
     "GuidanceRef",
     "GuidanceListingPage",
     "IDSAFetchError",
-    "IDSA_DATASET_DISPLAY_NAME",
-    "IDSA_DATASET_NAME",
-    "IDSAGuidelineListingPage",
     "IDSAGuidelineRef",
     "LinkMode",
-    "LISTING_URL",
     "NiceFetchError",
     "OutputFormat",
     "ScrapeError",
@@ -63,7 +52,6 @@
     "ScraperSource",
     "USER_AGENT",
     "absolute_unique_urls",
-    "build_idsa_guideline_text",
     "build_guideline_text",
     "clean_text",
     "document_title",
diff --git a/datasets/amfv_datasets/scraping/idsa.py b/datasets/amfv_datasets/scraping/idsa.py
index 9941b3f1..1feffcc5 100644
--- a/datasets/amfv_datasets/scraping/idsa.py
+++ b/datasets/amfv_datasets/scraping/idsa.py
@@ -21,10 +21,8 @@
 
 BASE_URL = "https://www.idsociety.org"
 LISTING_URL = f"{BASE_URL}/practice-guideline/all-practice-guidelines"
-IDSA_DATASET_NAME = "idsa-webscrape"
-IDSA_DATASET_DISPLAY_NAME = "IDSA Webscrape"
 DOCUMENT_DELAY_SECONDS = 5.0
-SHORT_CONTENT_CHARS = 10_000
+_SHORT_CONTENT_CHARS = 10_000
 
 _IDSA_PATH_RE = re.compile(r"^/practice-guideline/(?P<slug>[^/]+)(?:/|$)", re.IGNORECASE)
 _BACK_TO_TOP_RE = re.compile(r"^\s*back to top\s*$", re.IGNORECASE)
@@ -106,20 +104,12 @@ def list_practice_guidelines(
     return IDSAGuidelineListingPage(refs=refs, total=len(refs))
 
 
-def build_guideline_text(
+def _build_guideline_text(
     client: httpx.Client,
     ref: IDSAGuidelineRef,
     *,
     link_mode: LinkMode = LinkMode.KEEP,
 ) -> tuple[str, int, str, dict[str, list[str]]]:
-    """Scrape a guideline page into markdown text, section count, title, and links.
-
-    Args:
-        client: HTTP client used to fetch the guideline page.
-        ref: IDSA guideline reference to scrape.
-        link_mode: Whether links are kept as markdown links or stripped to their
-            visible text (default: LinkMode.KEEP).
-    """
     response = client.get(ref.page_url)
     response.raise_for_status()
     title = document_title(response.text, fallback=ref.title)
@@ -144,7 +134,7 @@ def scrape_guideline(
         link_mode: Whether links are kept as markdown links or stripped to their
             visible text (default: LinkMode.KEEP).
     """
-    content, section_count, title, links_metadata = build_guideline_text(client, ref, link_mode=link_mode)
+    content, section_count, title, links_metadata = _build_guideline_text(client, ref, link_mode=link_mode)
     return ScrapedDocument(
         source="idsa",
         external_id=f"idsa-{ref.slug}",
@@ -372,7 +362,7 @@ def _section_count(content_html: str) -> int:
 
 def _quality_flags(content: str) -> list[str]:
     flags: list[str] = []
-    if len(content) < SHORT_CONTENT_CHARS:
+    if len(content) < _SHORT_CONTENT_CHARS:
         flags.append("short_content")
     return flags
 
@@ -384,14 +374,10 @@ def _page_url(slug: str) -> str:
 __all__ = [
     "BASE_URL",
     "DOCUMENT_DELAY_SECONDS",
-    "IDSA_DATASET_DISPLAY_NAME",
-    "IDSA_DATASET_NAME",
     "IDSAFetchError",
     "IDSAGuidelineListingPage",
     "IDSAGuidelineRef",
     "LISTING_URL",
-    "SHORT_CONTENT_CHARS",
-    "build_guideline_text",
     "idsa_ref_from_url",
     "list_practice_guidelines",
     "scrape_guideline",
diff --git a/datasets/test/test_scraping_idsa.py b/datasets/test/test_scraping_idsa.py
index a83c4ae9..2c6073f7 100644
--- a/datasets/test/test_scraping_idsa.py
+++ b/datasets/test/test_scraping_idsa.py
@@ -9,19 +9,25 @@
     LISTING_URL,
     IDSAFetchError,
     IDSAGuidelineRef,
-    build_guideline_text,
     idsa_ref_from_url,
     list_practice_guidelines,
     scrape_guideline,
     scrape_idsa,
 )
 
+_LISTING_CASES = (
+    ("current-guideline", "Current Guideline", 2024, ("Current",)),
+    ("current-endorsed-guideline", "Current Endorsed Guideline", 2023, ("Current", "Endorsed")),
+    ("endorsed-guideline", "Endorsed Guideline", 2022, ("Endorsed",)),
+    ("archived-guideline", "Archived Guideline", 2021, ("Archived",)),
+    ("development-guideline", "Development Guideline", 2020, ("In Development",)),
+    ("archived-development-guideline", "Archived Development Guideline", 2019, ("Archived", "In Development")),
+)
 
-def test_list_practice_guidelines_parses_and_filters_default_statuses() -> None:
-    """The IDSA listing includes only current non-development guidelines by default."""
-    client = httpx.Client(transport=httpx.MockTransport(_listing_handler), base_url=BASE_URL)
 
-    listing = list_practice_guidelines(client)
+def test_list_practice_guidelines_parses_listing_fields() -> None:
+    """The IDSA listing parser normalizes title, URL, year, and statuses."""
+    listing = list_practice_guidelines(_listing_client())
 
     assert listing.total == 2
     assert listing.refs == [
@@ -42,45 +48,36 @@ def test_list_practice_guidelines_parses_and_filters_default_statuses() -> None:
     ]
 
 
-def test_list_practice_guidelines_can_include_archived_statuses() -> None:
-    """Archived guidelines are included only when requested."""
-    client = httpx.Client(transport=httpx.MockTransport(_listing_handler), base_url=BASE_URL)
-
-    listing = list_practice_guidelines(client, include_archived=True)
-
-    assert [ref.slug for ref in listing.refs] == [
-        "current-guideline",
-        "current-endorsed-guideline",
-        "archived-guideline",
-    ]
-
-
-def test_list_practice_guidelines_can_include_in_development_statuses() -> None:
-    """In-development guidelines are included only when requested."""
-    client = httpx.Client(transport=httpx.MockTransport(_listing_handler), base_url=BASE_URL)
-
-    listing = list_practice_guidelines(client, include_in_development=True)
-
-    assert [ref.slug for ref in listing.refs] == [
-        "current-guideline",
-        "current-endorsed-guideline",
-        "development-guideline",
-    ]
-
-
-def test_list_practice_guidelines_requires_both_flags_for_archived_development_statuses() -> None:
-    """Guidelines marked both archived and in development require both inclusion flags."""
-    client = httpx.Client(transport=httpx.MockTransport(_listing_handler), base_url=BASE_URL)
+@pytest.mark.parametrize(
+    ("kwargs", "expected_slugs"),
+    [
+        ({}, ["current-guideline", "current-endorsed-guideline"]),
+        (
+            {"include_archived": True},
+            ["current-guideline", "current-endorsed-guideline", "archived-guideline"],
+        ),
+        (
+            {"include_in_development": True},
+            ["current-guideline", "current-endorsed-guideline", "development-guideline"],
+        ),
+        (
+            {"include_archived": True, "include_in_development": True},
+            [
+                "current-guideline",
+                "current-endorsed-guideline",
+                "archived-guideline",
+                "development-guideline",
+                "archived-development-guideline",
+            ],
+        ),
+    ],
+    ids=["default", "include-archived", "include-in-development", "include-both"],
+)
+def test_list_practice_guidelines_filters_statuses(kwargs: dict[str, bool], expected_slugs: list[str]) -> None:
+    """Status flags preserve the intended IDSA listing inclusion policy."""
+    listing = list_practice_guidelines(_listing_client(), **kwargs)
 
-    listing = list_practice_guidelines(client, include_archived=True, include_in_development=True)
-
-    assert [ref.slug for ref in listing.refs] == [
-        "current-guideline",
-        "current-endorsed-guideline",
-        "archived-guideline",
-        "development-guideline",
-        "archived-development-guideline",
-    ]
+    assert [ref.slug for ref in listing.refs] == expected_slugs
 
 
 def test_idsa_ref_from_url_normalizes_practice_guideline_url() -> None:
@@ -110,71 +107,47 @@ def test_idsa_ref_from_url_rejects_non_guideline_urls(url: str) -> None:
         idsa_ref_from_url(url)
 
 
-def test_build_guideline_text_extracts_content_and_link_metadata() -> None:
+def test_scrape_guideline_extracts_content_and_link_metadata() -> None:
     """Guideline page content is converted to markdown and noisy UI is stripped."""
-    client = httpx.Client(transport=httpx.MockTransport(_guideline_handler), base_url=BASE_URL)
+    document = scrape_guideline(_guideline_client(_guideline_html()), _ref())
+
+    assert document.title == "Current Guideline"
+    assert document.section_count == 3
+    assert "# Current Guideline" in document.content
+    assert "## Abstract" in document.content
+    assert "## Recommendations" in document.content
+    assert "[Download PDF](https://academic.oup.com/example.pdf)" in document.content
+    assert "Recommendation text with [evidence](https://doi.org/10.1093/cid/example)." in document.content
+    assert "Back to top" not in document.content
+    assert "Table of Contents" not in document.content
+    assert "https://www.idsociety.org#abstract" not in document.content
+    assert document.metadata["external_links"] == [
+        "https://academic.oup.com/example.pdf",
+        "https://doi.org/10.1093/cid/example",
+    ]
+    assert document.metadata["pdf_links"] == ["https://academic.oup.com/example.pdf"]
+    assert document.metadata["content_length_chars"] == len(document.content)
+    assert document.metadata["quality_flags"] == ["short_content"]
 
-    content, section_count, title, links_metadata = build_guideline_text(
-        client,
-        IDSAGuidelineRef(
-            title="Current Guideline",
-            slug="current-guideline",
-            page_url="https://www.idsociety.org/practice-guideline/current-guideline/",
-            year=2024,
-            statuses=("Current",),
-        ),
-    )
 
-    assert title == "Current Guideline"
-    assert section_count == 3
-    assert "# Current Guideline" in content
-    assert "## Abstract" in content
-    assert "## Recommendations" in content
-    assert "Recommendation text with [evidence](https://doi.org/10.1093/cid/example)." in content
-    assert "Back to top" not in content
-    assert "Table of Contents" not in content
-    assert "https://www.idsociety.org#abstract" not in content
-    assert links_metadata == {
-        "external_links": [
-            "https://academic.oup.com/example.pdf",
-            "https://doi.org/10.1093/cid/example",
-        ],
-        "pdf_links": ["https://academic.oup.com/example.pdf"],
-    }
-
-    stripped_content, _section_count, _title, _links_metadata = build_guideline_text(
-        client,
-        IDSAGuidelineRef(
-            title="Current Guideline",
-            slug="current-guideline",
-            page_url="https://www.idsociety.org/practice-guideline/current-guideline/",
-            year=2024,
-            statuses=("Current",),
-        ),
-        link_mode=LinkMode.STRIP,
-    )
-    assert "Recommendation text with evidence." in stripped_content
+def test_scrape_guideline_strips_links_when_requested() -> None:
+    """The IDSA scraper honors the shared link-mode option."""
+    document = scrape_guideline(_guideline_client(_guideline_html()), _ref(), link_mode=LinkMode.STRIP)
+
+    assert "Recommendation text with evidence." in document.content
+    assert "[evidence]" not in document.content
 
 
 def test_scrape_idsa_returns_normalized_document() -> None:
     """The IDSA scraper returns normalized scraped documents."""
-
-    def handler(request: httpx.Request) -> httpx.Response:
-        if str(request.url) == LISTING_URL:
-            return httpx.Response(200, text=_listing_html())
-        if str(request.url) == "https://www.idsociety.org/practice-guideline/current-guideline/":
-            return httpx.Response(200, text=_guideline_html())
-        raise AssertionError(f"Unexpected URL: {request.url}")
-
-    original_client_factory = "amfv_datasets.scraping.idsa.default_client"
-    transport = httpx.MockTransport(handler)
+    transport = httpx.MockTransport(_scrape_idsa_handler)
 
     class ClientFactory:
         def __call__(self) -> httpx.Client:
             return httpx.Client(transport=transport)
 
     with pytest.MonkeyPatch.context() as monkeypatch:
-        monkeypatch.setattr(original_client_factory, ClientFactory())
+        monkeypatch.setattr("amfv_datasets.scraping.idsa.default_client", ClientFactory())
         scrape_run = scrape_idsa(documents=1)
         documents = list(scrape_run.documents)
 
@@ -184,111 +157,80 @@ def __call__(self) -> httpx.Client:
     assert documents[0].external_id == "idsa-current-guideline"
     assert documents[0].metadata["statuses"] == ["Current"]
     assert documents[0].metadata["year"] == 2024
-    assert documents[0].metadata["pdf_links"] == ["https://academic.oup.com/example.pdf"]
-    assert documents[0].metadata["content_length_chars"] == len(documents[0].content)
-    assert documents[0].metadata["quality_flags"] == ["short_content"]
+    assert documents[0].metadata["slug"] == "current-guideline"
+    assert documents[0].metadata["listing_url"] == LISTING_URL
 
 
-def test_scrape_guideline_does_not_flag_long_content() -> None:
-    """Long guideline content records length without short-content quality flags."""
+@pytest.mark.parametrize(
+    ("paragraph", "expected_flags"),
+    [
+        ("Useful abstract.", ["short_content"]),
+        ("Recommendation text. " * 700, []),
+    ],
+    ids=["short", "long"],
+)
+def test_scrape_guideline_sets_content_quality_flags(paragraph: str, expected_flags: list[str]) -> None:
+    """Short content is flagged while long content is left unflagged."""
+    document = scrape_guideline(_guideline_client(_guideline_html(paragraph=paragraph)), _ref())
 
-    def handler(request: httpx.Request) -> httpx.Response:
-        assert str(request.url) == "https://www.idsociety.org/practice-guideline/long-guideline/"
-        return httpx.Response(200, text=_long_guideline_html())
+    assert document.metadata["content_length_chars"] == len(document.content)
+    assert document.metadata["quality_flags"] == expected_flags
 
-    client = httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL)
 
-    document = scrape_guideline(
-        client,
-        IDSAGuidelineRef(
-            title="Long Guideline",
-            slug="long-guideline",
-            page_url="https://www.idsociety.org/practice-guideline/long-guideline/",
-            year=2024,
-            statuses=("Current",),
-        ),
-    )
+def _listing_client() -> httpx.Client:
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert str(request.url) == LISTING_URL
+        return httpx.Response(200, text=_listing_html())
 
-    assert document.metadata["content_length_chars"] == len(document.content)
-    assert document.metadata["quality_flags"] == []
+    return httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL)
 
 
-def _listing_handler(request: httpx.Request) -> httpx.Response:
-    assert str(request.url) == LISTING_URL
-    return httpx.Response(200, text=_listing_html())
+def _guideline_client(content_html: str) -> httpx.Client:
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert str(request.url) == "https://www.idsociety.org/practice-guideline/current-guideline/"
+        return httpx.Response(200, text=content_html)
 
+    return httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL)
 
-def _guideline_handler(request: httpx.Request) -> httpx.Response:
-    assert str(request.url) == "https://www.idsociety.org/practice-guideline/current-guideline/"
-    return httpx.Response(200, text=_guideline_html())
+
+def _scrape_idsa_handler(request: httpx.Request) -> httpx.Response:
+    if str(request.url) == LISTING_URL:
+        return httpx.Response(200, text=_listing_html())
+    if str(request.url) == "https://www.idsociety.org/practice-guideline/current-guideline/":
+        return httpx.Response(200, text=_guideline_html())
+    raise AssertionError(f"Unexpected URL: {request.url}")
 
 
 def _listing_html() -> str:
-    return """
-        <html>
-          <div class="alpha-listing">
-            <ul class="list-pages">
-              <li>
-                <ul class="list-pages__categories"><li class="category-dot category-dot-current">Current</li></ul>
-                <a class="list-pages__link" href="/practice-guideline/current-guideline/">Current Guideline</a>
-                <span class="list-pages__year">2024</span>
-              </li>
-              <li>
-                <ul class="list-pages__categories">
-                  <li class="category-dot category-dot-current">Current</li>
-                  <li class="category-dot category-dot-endorsed">Endorsed</li>
-                </ul>
-                <a class="list-pages__link" href="/practice-guideline/current-endorsed-guideline/">
-                  Current Endorsed Guideline
-                </a>
-                <span class="list-pages__year">2023</span>
-              </li>
-              <li>
-                <ul class="list-pages__categories"><li class="category-dot category-dot-endorsed">Endorsed</li></ul>
-                <a class="list-pages__link" href="/practice-guideline/endorsed-guideline/">Endorsed Guideline</a>
-                <span class="list-pages__year">2022</span>
-              </li>
-              <li>
-                <ul class="list-pages__categories"><li class="category-dot category-dot-archived">Archived</li></ul>
-                <a class="list-pages__link" href="/practice-guideline/archived-guideline/">Archived Guideline</a>
-                <span class="list-pages__year">2021</span>
-              </li>
-              <li>
-                <ul class="list-pages__categories">
-                  <li class="category-dot category-dot-in-development">In Development</li>
-                </ul>
-                <a class="list-pages__link" href="/practice-guideline/development-guideline/">
-                  Development Guideline
-                </a>
-                <span class="list-pages__year">2020</span>
-              </li>
-              <li>
-                <ul class="list-pages__categories">
-                  <li class="category-dot category-dot-archived">Archived</li>
-                  <li class="category-dot category-dot-in-development">In Development</li>
-                </ul>
-                <a class="list-pages__link" href="/practice-guideline/archived-development-guideline/">
-                  Archived Development Guideline
-                </a>
-                <span class="list-pages__year">2019</span>
-              </li>
-            </ul>
-          </div>
-        </html>
+    items = "\n".join(
+        _listing_item(slug=slug, title=title, year=year, statuses=statuses)
+        for slug, title, year, statuses in _LISTING_CASES
+    )
+    return f'<html><div class="alpha-listing"><ul class="list-pages">{items}</ul></div></html>'
+
+
+def _listing_item(*, slug: str, title: str, year: int, statuses: tuple[str, ...]) -> str:
+    categories = "".join(f'<li class="category-dot">{status}</li>' for status in statuses)
+    return f"""
+        <li>
+          <ul class="list-pages__categories">{categories}</ul>
+          <a class="list-pages__link" href="/practice-guideline/{slug}/">{title}</a>
+          <span class="list-pages__year">{year}</span>
+        </li>
     """
 
 
-def _guideline_html() -> str:
-    return """
+def _guideline_html(*, title: str = "Current Guideline", paragraph: str = "Useful abstract.") -> str:
+    return f"""
         <html>
-          <title>Current Guideline | IDSA</title>
+          <title>{title} | IDSA</title>
           <div class="standardpage-col-left">
             <nav>Navigation noise</nav>
             <p>Intro column noise.</p>
           </div>
           <div class="standardpage-col-left">
             <p><a href="https://academic.oup.com/example.pdf">Download PDF</a></p>
-            <h1>Current Guideline</h1>
+            <h1>{title}</h1>
             <h3>Table of Contents</h3>
             <ul>
               <li><a href="#abstract">Abstract</a></li>
@@ -297,7 +239,7 @@ def _guideline_html() -> str:
             <p class="social">Share this page</p>
             <p>Published January 1, 2024</p>
             <h2>Abstract</h2>
-            <p>Useful abstract.</p>
+            <p>{paragraph}</p>
             <h2>Recommendations</h2>
             <p>Recommendation text with <a href="https://doi.org/10.1093/cid/example">evidence</a>.</p>
             <p>Back to top</p>
@@ -306,15 +248,11 @@ def _guideline_html() -> str:
     """
 
 
-def _long_guideline_html() -> str:
-    long_text = "Recommendation text. " * 700
-    return f"""
-        <html>
-          <title>Long Guideline | IDSA</title>
-          <div class="standardpage-col-left">
-            <h1>Long Guideline</h1>
-            <h2>Recommendations</h2>
-            <p>{long_text}</p>
-          </div>
-        </html>
-    """
+def _ref() -> IDSAGuidelineRef:
+    return IDSAGuidelineRef(
+        title="Current Guideline",
+        slug="current-guideline",
+        page_url="https://www.idsociety.org/practice-guideline/current-guideline/",
+        year=2024,
+        statuses=("Current",),
+    )

From 54e9f712894f346ff4e5610c963bf40a9a901638 Mon Sep 17 00:00:00 2001
From: mkrastev <matey_krastev2@hotmail.com>
Date: Wed, 15 Jul 2026 13:18:03 +0300
Subject: [PATCH 04/15] add cps statement scraper

---
 datasets/amfv_datasets/scraping/__init__.py |  18 ++
 datasets/amfv_datasets/scraping/cli.py      |  38 ++-
 datasets/amfv_datasets/scraping/cps.py      | 285 ++++++++++++++++++++
 datasets/amfv_datasets/scraping/html.py     |   2 +
 datasets/test/test_scraping_cli.py          |  45 ++++
 datasets/test/test_scraping_cps.py          | 159 +++++++++++
 datasets/test/test_scraping_html.py         |   9 +
 7 files changed, 548 insertions(+), 8 deletions(-)
 create mode 100644 datasets/amfv_datasets/scraping/cps.py
 create mode 100644 datasets/test/test_scraping_cps.py

diff --git a/datasets/amfv_datasets/scraping/__init__.py b/datasets/amfv_datasets/scraping/__init__.py
index cd7cdb06..827edb37 100644
--- a/datasets/amfv_datasets/scraping/__init__.py
+++ b/datasets/amfv_datasets/scraping/__init__.py
@@ -9,6 +9,16 @@
     scrape_listing_documents,
 )
 from amfv_datasets.scraping.cli import OutputFormat, ScraperSource
+from amfv_datasets.scraping.cps import (
+    CpsFetchError,
+    CpsStatementRef,
+    build_statement_text,
+    list_statements,
+    listing_page_url,
+    scrape_cps,
+    scrape_statement,
+    statement_ref_from_url,
+)
 from amfv_datasets.scraping.html import (
     LinkMode,
     absolute_unique_urls,
@@ -31,6 +41,8 @@
 __all__ = [
     "GuidanceRef",
     "GuidanceListingPage",
+    "CpsFetchError",
+    "CpsStatementRef",
     "LinkMode",
     "NiceFetchError",
     "OutputFormat",
@@ -41,6 +53,7 @@
     "USER_AGENT",
     "absolute_unique_urls",
     "build_guideline_text",
+    "build_statement_text",
     "clean_text",
     "document_title",
     "default_client",
@@ -48,7 +61,12 @@
     "guidance_ref_from_url",
     "html_to_markdown",
     "list_published_guidance",
+    "list_statements",
+    "listing_page_url",
     "scrape_guideline",
     "scrape_listing_documents",
     "scrape_nice",
+    "scrape_cps",
+    "scrape_statement",
+    "statement_ref_from_url",
 ]
diff --git a/datasets/amfv_datasets/scraping/cli.py b/datasets/amfv_datasets/scraping/cli.py
index e8aece34..bb46d846 100644
--- a/datasets/amfv_datasets/scraping/cli.py
+++ b/datasets/amfv_datasets/scraping/cli.py
@@ -25,6 +25,7 @@
 )
 
 from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun
+from amfv_datasets.scraping.cps import scrape_cps
 from amfv_datasets.scraping.html import LinkMode
 from amfv_datasets.scraping.nice import scrape_nice
 
@@ -34,6 +35,7 @@ class ScraperSource(StrEnum):
 
     ALL = "all"
     NICE = "nice"
+    CPS = "cps"
 
 
 class OutputFormat(StrEnum):
@@ -68,13 +70,33 @@ def scrape_documents(
     if documents is not None and documents < 1:
         raise ValueError(f"documents must be at least 1; got {documents}")
 
-    for selected_source in _expand_source(source):
-        match selected_source:
-            case ScraperSource.NICE:
-                return scrape_nice(documents=documents, link_mode=link_mode, url=url)
-            case ScraperSource.ALL:
-                raise AssertionError("expanded source cannot be all")
-    raise AssertionError(f"unsupported source: {source}")
+    selected_sources = _expand_source(source)
+    if url is not None and len(selected_sources) != 1:
+        raise ValueError("--url requires one specific scraper source, not 'all'")
+    runs = tuple(
+        _scrape_source(selected_source, documents=documents, link_mode=link_mode, url=url)
+        for selected_source in selected_sources
+    )
+    if len(runs) == 1:
+        return runs[0]
+    total = sum(run.total for run in runs) if all(run.total is not None for run in runs) else None
+    return ScrapeRun(documents=(document for run in runs for document in run), total=total)
+
+
+def _scrape_source(
+    source: ScraperSource,
+    *,
+    documents: int | None,
+    link_mode: LinkMode,
+    url: str | None,
+) -> ScrapeRun:
+    match source:
+        case ScraperSource.NICE:
+            return scrape_nice(documents=documents, link_mode=link_mode, url=url)
+        case ScraperSource.CPS:
+            return scrape_cps(documents=documents, link_mode=link_mode, url=url)
+        case ScraperSource.ALL:
+            raise AssertionError("expanded source cannot be all")
 
 
 def write_jsonl(documents: Iterable[ScrapedDocument], output: TextIO) -> int:
@@ -125,7 +147,7 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path
 
 def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]:
     if source is ScraperSource.ALL:
-        return (ScraperSource.NICE,)
+        return (ScraperSource.NICE, ScraperSource.CPS)
     return (source,)
 
 
diff --git a/datasets/amfv_datasets/scraping/cps.py b/datasets/amfv_datasets/scraping/cps.py
new file mode 100644
index 00000000..4b9ba1e9
--- /dev/null
+++ b/datasets/amfv_datasets/scraping/cps.py
@@ -0,0 +1,285 @@
+"""Scrape Canadian Paediatric Society statements into normalized markdown.
+
+CPS retains copyright in its position statements and practice points. This
+module supports a permission-gated ingestion workflow; obtain written CPS
+permission before running a corpus scrape or redistributing its output.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from dataclasses import dataclass
+from urllib.parse import unquote, urljoin, urlparse
+
+import httpx
+from lxml import html as lxml_html
+from markdownify import MarkdownConverter
+
+from amfv_datasets.scraping.base import (
+    ScrapedDocument,
+    ScrapeError,
+    ScrapeRun,
+    default_client,
+    scrape_listing_documents,
+)
+from amfv_datasets.scraping.html import LinkMode, clean_text
+
+BASE_URL = "https://cps.ca"
+STATEMENTS_URL = f"{BASE_URL}/en/documents/statements-by-date"
+DOCUMENT_DELAY_SECONDS = 10.0
+LISTING_PAGE_SIZE = 10
+
+_POSITION_PATH_RE = re.compile(r"^/(?:en/)?documents/position/(?P<slug>[^/]+)/?$", re.IGNORECASE)
+_DATE_PATTERNS = {
+    "posted": re.compile(r"\bPosted:\s*([A-Za-z]+\s+\d{1,2},\s+\d{4})", re.IGNORECASE),
+    "reaffirmed": re.compile(r"\bReaffirmed:\s*([A-Za-z]+\s+\d{1,2},\s+\d{4})", re.IGNORECASE),
+    "updated": re.compile(r"\bUpdated:\s*([A-Za-z]+\s+\d{1,2},\s+\d{4})", re.IGNORECASE),
+}
+_BLANK_LINES_RE = re.compile(r"\n{3,}")
+_EMPTY_MARKDOWN_LINK_RE = re.compile(r"(?<!!)\[\]\([^)]*\)")
+_DECORATIVE_IMAGE_FILENAMES = {"file-pdf.svg"}
+
+
+class _CpsMarkdownConverter(MarkdownConverter):
+    """Preserve CPS superscripts that carry clinical meaning or citations."""
+
+    def convert_a(self, element, text: str, parent_tags: set[str]) -> str:  # noqa: ANN001
+        classes = element.get("class") or ()
+        href = element.get("href")
+        if "reference" in classes and href and "#ref" in href:
+            marker = element.get_text(strip=True).strip("[]")
+            if "a" in (self.options.get("strip") or ()):
+                return f"[{marker}]"
+            return f"[{marker}]({href})"
+        return super().convert_a(element, text, parent_tags)
+
+    def convert_sup(self, element, text: str, parent_tags: set[str]) -> str:  # noqa: ANN001
+        if not text.strip():
+            return ""
+        citation = element.find("a", href=lambda href: href and "#ref" in href)
+        if citation:
+            marker = element.get_text(strip=True).strip("[]")
+            if "a" in (self.options.get("strip") or ()):
+                return f"[{marker}]"
+            return f"[{marker}]({citation.get('href')})"
+        return f"<sup>{text}</sup>"
+
+
+_CPS_MARKDOWN_CONVERTERS = {
+    LinkMode.KEEP: _CpsMarkdownConverter(bullets="-", heading_style="ATX"),
+    LinkMode.STRIP: _CpsMarkdownConverter(bullets="-", heading_style="ATX", strip=("a",)),
+}
+
+
+class CpsFetchError(ScrapeError):
+    """Raised when a CPS statement cannot be discovered or parsed."""
+
+
+@dataclass(frozen=True)
+class CpsStatementRef:
+    """A CPS statement or practice point discovered from the date index."""
+
+    slug: str
+    title: str
+    page_url: str
+
+
+def statement_ref_from_url(url: str, *, title: str | None = None) -> CpsStatementRef:
+    """Parse a CPS position-statement URL into a canonical reference."""
+    parsed = urlparse(url.strip())
+    if parsed.scheme not in {"http", "https"} or parsed.netloc.lower() not in {"cps.ca", "www.cps.ca"}:
+        raise CpsFetchError(f"Enter a CPS statement URL from cps.ca; got {url!r}")
+
+    match = _POSITION_PATH_RE.match(unquote(parsed.path))
+    if not match:
+        raise CpsFetchError(f"Enter a URL like https://cps.ca/en/documents/position/example-statement; got {url!r}")
+    slug = match.group("slug")
+    return CpsStatementRef(
+        slug=slug,
+        title=title or slug.replace("-", " "),
+        page_url=f"{BASE_URL}/en/documents/position/{slug}",
+    )
+
+
+def listing_page_url(page: int) -> str:
+    """Return the CPS date-index URL for a one-based page number."""
+    if page < 1:
+        raise ValueError(f"page must be at least 1; got {page}")
+    if page == 1:
+        return STATEMENTS_URL
+    return f"{STATEMENTS_URL}/P{(page - 1) * LISTING_PAGE_SIZE}"
+
+
+def list_statements(client: httpx.Client, page: int) -> list[CpsStatementRef]:
+    """Return unique statement references from one CPS date-index page."""
+    response = client.get(listing_page_url(page))
+    response.raise_for_status()
+    doc = lxml_html.fromstring(response.text)
+    anchors = doc.xpath("//div[contains(concat(' ', normalize-space(@class), ' '), ' stmt-title ')]//a[@href]")
+
+    refs: list[CpsStatementRef] = []
+    seen_urls: set[str] = set()
+    for anchor in anchors:
+        href = anchor.get("href")
+        title = clean_text(anchor.text_content(), drop_numeric_citations=False)
+        if not href or not title:
+            continue
+        try:
+            ref = statement_ref_from_url(urljoin(BASE_URL, href), title=title)
+        except CpsFetchError:
+            continue
+        if ref.page_url in seen_urls:
+            continue
+        seen_urls.add(ref.page_url)
+        refs.append(ref)
+    return refs
+
+
+def _content_root(html_text: str) -> lxml_html.HtmlElement:
+    doc = lxml_html.fromstring(html_text)
+    selectors = (
+        "//div[contains(concat(' ', normalize-space(@class), ' '), ' statement-wrapper ')][1]",
+        "//main//article[1]",
+        "//*[@id='main-content']//article[1]",
+        "//*[@id='main-content'][1]",
+        "//main[1]",
+    )
+    for selector in selectors:
+        matches = doc.xpath(selector)
+        if matches:
+            return matches[0]
+    raise CpsFetchError("No readable statement content found on the CPS page")
+
+
+def _normalize_content(root: lxml_html.HtmlElement, *, base_url: str) -> None:
+    removable = root.xpath(
+        ".//script | .//style | .//nav | .//form | .//button | .//noscript | "
+        ".//*[contains(concat(' ', normalize-space(@class), ' '), ' breadcrumb ')] | "
+        ".//*[contains(@class, 'share')] | "
+        ".//*[contains(@class, 'social')] | "
+        ".//*[contains(concat(' ', normalize-space(@class), ' '), ' related-content ')] | "
+        ".//*[contains(concat(' ', normalize-space(@class), ' '), ' sidebar ')] | "
+        ".//*[contains(concat(' ', normalize-space(@class), ' '), ' hide-for-print ')] | "
+        ".//*[contains(concat(' ', normalize-space(@class), ' '), ' show-for-print ')] | "
+        ".//*[contains(@class, 'print:tw-hidden')] | "
+        ".//*[contains(concat(' ', normalize-space(@class), ' '), ' --podcast ')]"
+    )
+    for element in removable:
+        element.drop_tree()
+
+    for link in root.xpath(".//a[contains(@href, '/en/education/test-your-knowledge')]"):
+        link.drop_tree()
+
+    for image in root.xpath(".//img[@src]"):
+        filename = urlparse(image.get("src")).path.rsplit("/", 1)[-1].lower()
+        if filename in _DECORATIVE_IMAGE_FILENAMES:
+            image.drop_tree()
+
+    for link in root.xpath(".//a[@href]"):
+        link.set("href", urljoin(base_url, link.get("href")))
+    for image in root.xpath(".//img[@src]"):
+        image.set("src", urljoin(base_url, image.get("src")))
+
+    for nested in root.xpath(".//strong//strong | .//em//em"):
+        nested.drop_tag()
+    for title in root.xpath(".//h1"):
+        title.drop_tree()
+    for heading in root.xpath(".//*[self::h1 or self::h2 or self::h3 or self::h4 or self::h5 or self::h6]"):
+        for emphasis in heading.xpath(".//strong | .//em | .//b | .//i"):
+            emphasis.drop_tag()
+        if not clean_text(heading.text_content(), drop_numeric_citations=False) and not heading.xpath(".//img"):
+            heading.drop_tree()
+
+
+def _content_to_markdown(root: lxml_html.HtmlElement, *, link_mode: LinkMode) -> str:
+    source = lxml_html.tostring(root, encoding="unicode")
+    markdown = _CPS_MARKDOWN_CONVERTERS[link_mode].convert(source)
+    markdown = _EMPTY_MARKDOWN_LINK_RE.sub("", markdown)
+    lines = [line.rstrip() for line in markdown.splitlines()]
+    return _BLANK_LINES_RE.sub("\n\n", "\n".join(lines)).strip()
+
+
+def build_statement_text(
+    html_text: str,
+    *,
+    link_mode: LinkMode = LinkMode.KEEP,
+    base_url: str = BASE_URL,
+) -> tuple[str, int]:
+    """Extract one CPS statement as markdown and return its section count."""
+    root = _content_root(html_text)
+    _normalize_content(root, base_url=base_url)
+    section_count = max(1, len(root.xpath(".//*[self::h2 or self::h3 or self::h4 or self::h5 or self::h6]")))
+    content = _content_to_markdown(root, link_mode=link_mode)
+    if not content:
+        raise CpsFetchError("No readable statement content found on the CPS page")
+    return content, section_count
+
+
+def scrape_statement(
+    client: httpx.Client,
+    ref: CpsStatementRef,
+    *,
+    link_mode: LinkMode = LinkMode.KEEP,
+) -> ScrapedDocument:
+    """Scrape one CPS statement into the shared document schema."""
+    response = client.get(ref.page_url)
+    response.raise_for_status()
+    content, section_count = build_statement_text(response.text, link_mode=link_mode, base_url=ref.page_url)
+    raw_root = _content_root(response.text)
+    headings = [clean_text(value) for value in raw_root.xpath(".//h1[1]//text()")]
+    title = " ".join(value for value in headings if value) or ref.title
+    metadata: dict[str, str] = {"slug": ref.slug}
+    visible_text = clean_text(raw_root.text_content(), drop_numeric_citations=False)
+    for key, pattern in _DATE_PATTERNS.items():
+        if match := pattern.search(visible_text):
+            metadata[key] = match.group(1).strip()
+    return ScrapedDocument(
+        source="cps",
+        external_id=f"cps-{ref.slug.lower()}",
+        title=title,
+        url=ref.page_url,
+        content=content,
+        section_count=section_count,
+        metadata=metadata,
+    )
+
+
+def scrape_cps(
+    *,
+    documents: int | None,
+    link_mode: LinkMode = LinkMode.KEEP,
+    url: str | None = None,
+) -> ScrapeRun:
+    """Configure a CPS scrape from one URL or the current-statements index."""
+    if url is not None:
+
+        def scrape_url() -> Iterable[ScrapedDocument]:
+            with default_client() as client:
+                yield scrape_statement(client, statement_ref_from_url(url), link_mode=link_mode)
+
+        return ScrapeRun(documents=scrape_url(), total=1)
+
+    return ScrapeRun(
+        documents=scrape_listing_documents(
+            documents=documents,
+            client_factory=default_client,
+            list_page=list_statements,
+            scrape_item=lambda client, ref: scrape_statement(client, ref, link_mode=link_mode),
+            document_delay_seconds=DOCUMENT_DELAY_SECONDS,
+        )
+    )
+
+
+__all__ = [
+    "BASE_URL",
+    "CpsFetchError",
+    "CpsStatementRef",
+    "STATEMENTS_URL",
+    "build_statement_text",
+    "list_statements",
+    "listing_page_url",
+    "scrape_cps",
+    "scrape_statement",
+    "statement_ref_from_url",
+]
diff --git a/datasets/amfv_datasets/scraping/html.py b/datasets/amfv_datasets/scraping/html.py
index 2bd1c968..15805281 100644
--- a/datasets/amfv_datasets/scraping/html.py
+++ b/datasets/amfv_datasets/scraping/html.py
@@ -119,6 +119,8 @@ def _absolutize_links(html_text: str, *, base_url: str) -> str:
     root = lxml_html.fragment_fromstring(html_text, create_parent="div")
     for link in root.xpath(".//a[@href]"):
         link.set("href", urljoin(base_url, link.get("href")))
+    for image in root.xpath(".//img[@src]"):
+        image.set("src", urljoin(base_url, image.get("src")))
     return "".join(lxml_html.tostring(child, encoding="unicode") for child in root)
 
 
diff --git a/datasets/test/test_scraping_cli.py b/datasets/test/test_scraping_cli.py
index 44f714fa..75239b64 100644
--- a/datasets/test/test_scraping_cli.py
+++ b/datasets/test/test_scraping_cli.py
@@ -11,6 +11,7 @@
 from amfv_datasets.scraping.cli import (
     ScraperSource,
     app,
+    scrape_documents,
     write_huggingface_dataset,
     write_jsonl,
     write_markdown_files,
@@ -219,6 +220,50 @@ def fake_scrape_documents(
     assert json.loads(result.stdout.splitlines()[0])["external_id"] == "nice-ng1"
 
 
+def test_scrape_documents_dispatches_cps(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The CPS source delegates to its source-specific scraper."""
+    calls = 0
+
+    def fake_scrape_cps(*, documents: int | None, link_mode: LinkMode, url: str | None) -> ScrapeRun:
+        nonlocal calls
+        calls += 1
+        assert documents == 2
+        assert link_mode is LinkMode.KEEP
+        assert url == "https://cps.ca/en/documents/position/example"
+        return ScrapeRun([_document()], total=1)
+
+    monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_cps", fake_scrape_cps)
+
+    run = scrape_documents(
+        ScraperSource.CPS,
+        documents=2,
+        link_mode=LinkMode.KEEP,
+        url="https://cps.ca/en/documents/position/example",
+    )
+
+    assert calls == 1
+    assert run.total == 1
+
+
+def test_scrape_documents_combines_all_sources(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The all source streams NICE and CPS documents with a combined total."""
+    nice = _document()
+    cps = ScrapedDocument("cps", "cps-example", "CPS example", "https://cps.ca/example", "content")
+    monkeypatch.setattr(
+        "amfv_datasets.scraping.cli.scrape_nice",
+        lambda **_kwargs: ScrapeRun([nice], total=1),
+    )
+    monkeypatch.setattr(
+        "amfv_datasets.scraping.cli.scrape_cps",
+        lambda **_kwargs: ScrapeRun([cps], total=1),
+    )
+
+    run = scrape_documents(ScraperSource.ALL, documents=1, link_mode=LinkMode.KEEP)
+
+    assert run.total == 2
+    assert [document.source for document in run] == ["nice", "cps"]
+
+
 def _document() -> ScrapedDocument:
     return ScrapedDocument(
         source="nice",
diff --git a/datasets/test/test_scraping_cps.py b/datasets/test/test_scraping_cps.py
new file mode 100644
index 00000000..26d3bd93
--- /dev/null
+++ b/datasets/test/test_scraping_cps.py
@@ -0,0 +1,159 @@
+"""Tests for Canadian Paediatric Society statement scraping helpers."""
+
+import httpx
+import pytest
+
+from amfv_datasets.scraping.cps import (
+    BASE_URL,
+    STATEMENTS_URL,
+    CpsFetchError,
+    CpsStatementRef,
+    list_statements,
+    listing_page_url,
+    scrape_statement,
+    statement_ref_from_url,
+)
+
+
+def test_listing_page_url_uses_cps_offsets() -> None:
+    """One-based pages map to the CPS P30 offset convention."""
+    assert listing_page_url(1) == STATEMENTS_URL
+    assert listing_page_url(2) == f"{STATEMENTS_URL}/P10"
+    assert listing_page_url(4) == f"{STATEMENTS_URL}/P30"
+
+
+def test_list_statements_discovers_unique_position_pages() -> None:
+    """The index yields unique position statements and ignores unrelated links."""
+    html_text = """
+    <html><div class="cell main-body">
+      <div class="stmt-title"><a href="/en/documents/position/acute-asthma">Acute asthma</a></div>
+      <div class="stmt-title"><a href="https://cps.ca/documents/position/acute-asthma">Duplicate asthma</a></div>
+      <div class="stmt-title"><a href="/en/documents/position/newborn-glucose">Newborn glucose</a></div>
+      <a href="/en/documents/about-position-statements">About statements</a>
+    </div><footer><a href="/en/documents/position/footer-link">Footer</a></footer></html>
+    """
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert str(request.url) == STATEMENTS_URL
+        return httpx.Response(200, text=html_text)
+
+    with httpx.Client(transport=httpx.MockTransport(handler)) as client:
+        assert list_statements(client, 1) == [
+            CpsStatementRef("acute-asthma", "Acute asthma", f"{BASE_URL}/en/documents/position/acute-asthma"),
+            CpsStatementRef(
+                "newborn-glucose",
+                "Newborn glucose",
+                f"{BASE_URL}/en/documents/position/newborn-glucose",
+            ),
+        ]
+
+
+def test_statement_ref_from_url_normalizes_supported_routes() -> None:
+    """English and language-neutral CPS routes become canonical HTTPS URLs."""
+    assert statement_ref_from_url("http://www.cps.ca/documents/position/Acute-Asthma?print=1#dose") == (
+        CpsStatementRef(
+            "Acute-Asthma",
+            "Acute Asthma",
+            "https://cps.ca/en/documents/position/Acute-Asthma",
+        )
+    )
+
+
+def test_statement_ref_from_url_rejects_non_statement_pages() -> None:
+    """CPS navigation pages cannot be mistaken for clinical statements."""
+    with pytest.raises(CpsFetchError):
+        statement_ref_from_url("https://cps.ca/en/documents")
+
+
+def test_scrape_statement_preserves_clinical_structure_and_metadata() -> None:
+    """Statement content is normalized while navigation and sharing chrome are removed."""
+    html_text = """
+    <html>
+      <head><title>Fallback | Canadian Paediatric Society</title></head>
+      <body>
+        <header><h1>Site heading</h1></header>
+        <main id="main-content">
+          <div class="statement-wrapper">
+            <div class="breadcrumb">Home / Documents</div>
+            <p>Position statement</p>
+            <h1>Management of well-appearing febrile young infants</h1>
+            <p>Posted: Oct 27, 2023 | Reaffirmed: Jan 12, 2026 | Updated: May 27, 2026</p>
+            <h2>Recommendations</h2>
+            <ul><li>Assess the infant:<ul><li>Check vital signs.</li></ul></li></ul>
+            <p>Use the <a href="/en/tools/risk-calculator">risk calculator</a>.</p>
+            <p>Current citation<a class="reference" href="#ref1">[1]</a>.</p>
+            <p>Legacy citation<sup>[<a class="reference" href="#ref2">2</a>]</sup>.</p>
+            <p>Maternal fever above 38<sup>o</sup>C and counts of 10<sup>9</sup>/L require attention.</p>
+            <p><a href="/documents/full-statement.pdf">Full statement PDF</a>
+              <img src="/assets/img/file-pdf.svg" alt="PDF icon"></p>
+            <p><a href="/empty-target"></a></p>
+            <table><tr><th>Age</th><th>Action</th></tr><tr><td>0-28 days</td><td>Investigate</td></tr></table>
+            <h2><strong>Algorithm</strong></h2>
+            <p><img src="/uploads/flowcharts/febrile-infant.png" alt=""></p>
+            <h2>References</h2>
+            <div class="reference-print-style"><ol>
+              <li><a name="ref1"></a>Reference one.</li>
+              <li><a name="ref2"></a>Reference two.</li>
+            </ol></div>
+            <a href="/en/education/test-your-knowledge"><img src="/assets/img/test.png" alt="Quiz"></a>
+            <div class="share-tools">Share this statement</div>
+            <aside class="related-content">Related news</aside>
+            <script>window.track = true;</script>
+          </div>
+        </main>
+        <footer>Contact CPS</footer>
+      </body>
+    </html>
+    """
+    ref = CpsStatementRef(
+        "febrile-young-infants",
+        "Listing title",
+        f"{BASE_URL}/en/documents/position/febrile-young-infants",
+    )
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert str(request.url) == ref.page_url
+        return httpx.Response(200, text=html_text)
+
+    with httpx.Client(transport=httpx.MockTransport(handler)) as client:
+        document = scrape_statement(client, ref)
+
+    assert document.source == "cps"
+    assert document.external_id == "cps-febrile-young-infants"
+    assert document.title == "Management of well-appearing febrile young infants"
+    assert document.section_count == 3
+    assert document.metadata == {
+        "slug": "febrile-young-infants",
+        "posted": "Oct 27, 2023",
+        "reaffirmed": "Jan 12, 2026",
+        "updated": "May 27, 2026",
+    }
+    assert "- Assess the infant:\n  - Check vital signs." in document.content
+    assert "[risk calculator](https://cps.ca/en/tools/risk-calculator)" in document.content
+    assert (
+        "Current citation[1](https://cps.ca/en/documents/position/febrile-young-infants#ref1)."
+        in document.content
+    )
+    assert (
+        "Legacy citation[2](https://cps.ca/en/documents/position/febrile-young-infants#ref2)."
+        in document.content
+    )
+    assert "38<sup>o</sup>C" in document.content
+    assert "10<sup>9</sup>/L" in document.content
+    assert "# Management of well-appearing" not in document.content
+    assert "## Algorithm" in document.content
+    assert "## **Algorithm**" not in document.content
+    assert "| Age | Action |" in document.content
+    assert "![](https://cps.ca/uploads/flowcharts/febrile-infant.png)" in document.content
+    assert "## References" in document.content
+    assert "1. Reference one." in document.content
+    assert "2. Reference two." in document.content
+    assert "[Full statement PDF](https://cps.ca/documents/full-statement.pdf)" in document.content
+    assert "file-pdf.svg" not in document.content
+    assert "empty-target" not in document.content
+    assert "Share this statement" not in document.content
+    assert "Related news" not in document.content
+    assert "test-your-knowledge" not in document.content
+    assert "assets/img/test.png" not in document.content
+    assert "window.track" not in document.content
+    assert "Contact CPS" not in document.content
diff --git a/datasets/test/test_scraping_html.py b/datasets/test/test_scraping_html.py
index d34a8fa3..6e82ae36 100644
--- a/datasets/test/test_scraping_html.py
+++ b/datasets/test/test_scraping_html.py
@@ -68,3 +68,12 @@ def test_html_to_markdown_can_strip_links() -> None:
     html_text = '<p>Offer <a href="https://example.org">treatment</a>.</p>'
 
     assert html_to_markdown(html_text, link_mode=LinkMode.STRIP) == "Offer treatment."
+
+
+def test_html_to_markdown_absolutizes_remote_image_references() -> None:
+    """Relative image references remain usable outside the source website."""
+    html_text = '<p><img src="/uploads/flowchart.png" alt="Treatment flowchart"></p>'
+
+    assert html_to_markdown(html_text, base_url="https://example.org") == (
+        "![Treatment flowchart](https://example.org/uploads/flowchart.png)"
+    )

From 059ab4817babe16e7b440e4c95a4e39c3b5b71b6 Mon Sep 17 00:00:00 2001
From: mkrastev <matey_krastev2@hotmail.com>
Date: Wed, 15 Jul 2026 13:18:09 +0300
Subject: [PATCH 05/15] add rch guideline scraper

---
 datasets/amfv_datasets/scraping/__init__.py |   8 +
 datasets/amfv_datasets/scraping/base.py     |  14 +-
 datasets/amfv_datasets/scraping/cli.py      | 105 +++++-
 datasets/amfv_datasets/scraping/html.py     |   2 +
 datasets/amfv_datasets/scraping/rch.py      | 398 ++++++++++++++++++++
 datasets/test/test_scraping_base.py         |  40 ++
 datasets/test/test_scraping_cli.py          |  87 ++++-
 datasets/test/test_scraping_html.py         |   9 +
 datasets/test/test_scraping_rch.py          | 252 +++++++++++++
 9 files changed, 894 insertions(+), 21 deletions(-)
 create mode 100644 datasets/amfv_datasets/scraping/rch.py
 create mode 100644 datasets/test/test_scraping_rch.py

diff --git a/datasets/amfv_datasets/scraping/__init__.py b/datasets/amfv_datasets/scraping/__init__.py
index cd7cdb06..6e111130 100644
--- a/datasets/amfv_datasets/scraping/__init__.py
+++ b/datasets/amfv_datasets/scraping/__init__.py
@@ -27,6 +27,11 @@
     scrape_guideline,
     scrape_nice,
 )
+from amfv_datasets.scraping.rch import (
+    RchFetchError,
+    RchGuidelineRef,
+    scrape_rch,
+)
 
 __all__ = [
     "GuidanceRef",
@@ -34,6 +39,8 @@
     "LinkMode",
     "NiceFetchError",
     "OutputFormat",
+    "RchFetchError",
+    "RchGuidelineRef",
     "ScrapeError",
     "ScrapeRun",
     "ScrapedDocument",
@@ -51,4 +58,5 @@
     "scrape_guideline",
     "scrape_listing_documents",
     "scrape_nice",
+    "scrape_rch",
 ]
diff --git a/datasets/amfv_datasets/scraping/base.py b/datasets/amfv_datasets/scraping/base.py
index 0708ee87..3651cdb7 100644
--- a/datasets/amfv_datasets/scraping/base.py
+++ b/datasets/amfv_datasets/scraping/base.py
@@ -74,7 +74,7 @@ def scrape_listing_documents[ListingItemT](
     documents: int | None,
     client_factory: Callable[[], AbstractContextManager[httpx.Client]],
     list_page: Callable[[httpx.Client, int], Iterable[ListingItemT]],
-    scrape_item: Callable[[httpx.Client, ListingItemT], ScrapedDocument],
+    scrape_item: Callable[[httpx.Client, ListingItemT], ScrapedDocument | None],
     document_delay_seconds: float = 5.0,
     first_page_items: Iterable[ListingItemT] | None = None,
 ) -> Iterable[ScrapedDocument]:
@@ -85,7 +85,8 @@ def scrape_listing_documents[ListingItemT](
             fetched until a page returns no items (default: None).
         client_factory: Factory returning a context-managed HTTP client.
         list_page: Function that lists source-specific items for a page.
-        scrape_item: Function that scrapes one listed item into a document.
+        scrape_item: Function that scrapes one listed item into a document. It
+            may return None to skip a discovered item that is not a document.
         document_delay_seconds: Delay before scraping each document after the
             first one (default: 5.0).
         first_page_items: Already-fetched first listing page items. When set,
@@ -99,6 +100,7 @@ def scrape_listing_documents[ListingItemT](
     with client_factory() as client:
         page = 1
         scraped = 0
+        attempted = 0
         page_items = list(first_page_items) if first_page_items is not None else None
         while documents is None or scraped < documents:
             if page_items is None:
@@ -111,9 +113,13 @@ def scrape_listing_documents[ListingItemT](
             for item in items:
                 if documents is not None and scraped >= documents:
                     break
-                if scraped and document_delay_seconds:
+                if attempted and document_delay_seconds:
                     time.sleep(document_delay_seconds)
-                yield scrape_item(client, item)
+                document = scrape_item(client, item)
+                attempted += 1
+                if document is None:
+                    continue
+                yield document
                 scraped += 1
             page += 1
 
diff --git a/datasets/amfv_datasets/scraping/cli.py b/datasets/amfv_datasets/scraping/cli.py
index e8aece34..1660d08a 100644
--- a/datasets/amfv_datasets/scraping/cli.py
+++ b/datasets/amfv_datasets/scraping/cli.py
@@ -5,7 +5,7 @@
 import json
 import re
 import sys
-from collections.abc import Iterable
+from collections.abc import Collection, Iterable
 from dataclasses import asdict
 from enum import StrEnum
 from pathlib import Path
@@ -27,6 +27,7 @@
 from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun
 from amfv_datasets.scraping.html import LinkMode
 from amfv_datasets.scraping.nice import scrape_nice
+from amfv_datasets.scraping.rch import scrape_rch
 
 
 class ScraperSource(StrEnum):
@@ -34,6 +35,7 @@ class ScraperSource(StrEnum):
 
     ALL = "all"
     NICE = "nice"
+    RCH = "rch"
 
 
 class OutputFormat(StrEnum):
@@ -53,6 +55,7 @@ def scrape_documents(
     documents: int | None,
     link_mode: LinkMode,
     url: str | None = None,
+    skip_urls: Collection[str] = (),
 ) -> ScrapeRun:
     """Configure a scrape for a source.
 
@@ -64,17 +67,44 @@ def scrape_documents(
         link_mode: Whether links are kept as markdown links or stripped to their
             visible text.
         url: Source URL to scrape as a single document (default: None).
+        skip_urls: Canonical document URLs already written by a previous run
+            (default: ()).
     """
     if documents is not None and documents < 1:
         raise ValueError(f"documents must be at least 1; got {documents}")
 
-    for selected_source in _expand_source(source):
-        match selected_source:
-            case ScraperSource.NICE:
-                return scrape_nice(documents=documents, link_mode=link_mode, url=url)
-            case ScraperSource.ALL:
-                raise AssertionError("expanded source cannot be all")
-    raise AssertionError(f"unsupported source: {source}")
+    selected_sources = _expand_source(source)
+    if url is not None and len(selected_sources) != 1:
+        raise ValueError("--url requires one specific scraper source, not 'all'")
+    runs = tuple(
+        _scrape_source(selected_source, documents=documents, link_mode=link_mode, url=url, skip_urls=skip_urls)
+        for selected_source in selected_sources
+    )
+    if len(runs) == 1:
+        return runs[0]
+    total = sum(run.total for run in runs) if all(run.total is not None for run in runs) else None
+    return ScrapeRun(documents=(document for run in runs for document in run), total=total)
+
+
+def _scrape_source(
+    source: ScraperSource,
+    *,
+    documents: int | None,
+    link_mode: LinkMode,
+    url: str | None,
+    skip_urls: Collection[str],
+) -> ScrapeRun:
+    match source:
+        case ScraperSource.NICE:
+            if skip_urls:
+                raise ValueError("resume is not implemented for the NICE scraper")
+            return scrape_nice(documents=documents, link_mode=link_mode, url=url)
+        case ScraperSource.RCH:
+            if skip_urls:
+                return scrape_rch(documents=documents, link_mode=link_mode, url=url, skip_urls=skip_urls)
+            return scrape_rch(documents=documents, link_mode=link_mode, url=url)
+        case ScraperSource.ALL:
+            raise AssertionError("expanded source cannot be all")
 
 
 def write_jsonl(documents: Iterable[ScrapedDocument], output: TextIO) -> int:
@@ -125,7 +155,7 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path
 
 def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]:
     if source is ScraperSource.ALL:
-        return (ScraperSource.NICE,)
+        return (ScraperSource.NICE, ScraperSource.RCH)
     return (source,)
 
 
@@ -138,6 +168,7 @@ def run(
     output_format: Annotated[OutputFormat, typer.Option("--format", "-f", help="Output format.")] = OutputFormat.JSONL,
     output_path: Annotated[Path | None, typer.Option("--output", "-o", help="Output JSONL file, markdown directory, or Hugging Face dataset directory. JSONL defaults to stdout.")] = None,  # noqa: E501
     progress: Annotated[bool, typer.Option("--progress/--no-progress", help="Show a Rich progress bar.")] = True,
+    resume: Annotated[bool, typer.Option("--resume", help="Append to JSONL and skip URLs already present.")] = False,
 ) -> None:  # fmt: skip
     """Run a scraper and write the scraped documents.
 
@@ -152,14 +183,25 @@ def run(
             dataset directory. When unset, JSONL is written to stdout (default:
             None).
         progress: Whether to show a Rich progress bar (default: True).
+        resume: Whether to append to an existing JSONL file and skip its
+            completed URLs (default: False).
     """
     parsed_documents = _parse_documents(documents)
-    scrape_run = scrape_documents(source, documents=parsed_documents, link_mode=link_mode, url=url)
+    completed_urls = _resume_urls(output_format, output_path, resume=resume)
+    scrape_kwargs = {
+        "documents": parsed_documents,
+        "link_mode": link_mode,
+        "url": url,
+    }
+    if completed_urls:
+        scrape_run = scrape_documents(source, skip_urls=completed_urls, **scrape_kwargs)
+    else:
+        scrape_run = scrape_documents(source, **scrape_kwargs)
     scraped_documents = scrape_run.documents
     if progress:
         scraped_documents = _progress_documents(scraped_documents, total=scrape_run.total)
     if output_format is OutputFormat.JSONL:
-        count = _write_jsonl_output(scraped_documents, output_path)
+        count = _write_jsonl_output(scraped_documents, output_path, append=resume)
     elif output_format is OutputFormat.HUGGINGFACE:
         if output_path is None:
             raise typer.BadParameter("--output is required when --format huggingface")
@@ -204,13 +246,50 @@ def _progress_documents(
             yield document
 
 
-def _write_jsonl_output(documents: Iterable[ScrapedDocument], output_path: Path | None) -> int:
+def _write_jsonl_output(
+    documents: Iterable[ScrapedDocument],
+    output_path: Path | None,
+    *,
+    append: bool = False,
+) -> int:
     if output_path is None:
         return write_jsonl(documents, sys.stdout)
-    with output_path.open("w", encoding="utf-8") as output:
+    with output_path.open("a" if append else "w", encoding="utf-8") as output:
         return write_jsonl(documents, output)
 
 
+def _resume_urls(
+    output_format: OutputFormat,
+    output_path: Path | None,
+    *,
+    resume: bool,
+) -> frozenset[str]:
+    if not resume:
+        return frozenset()
+    if output_format is not OutputFormat.JSONL or output_path is None:
+        raise typer.BadParameter("--resume requires --format jsonl and --output")
+    if not output_path.exists():
+        return frozenset()
+
+    urls: set[str] = set()
+    with output_path.open(encoding="utf-8") as existing:
+        for line_number, line in enumerate(existing, start=1):
+            try:
+                row = json.loads(line)
+                url = row["url"]
+            except (json.JSONDecodeError, KeyError, TypeError) as exc:
+                raise typer.BadParameter(
+                    f"cannot resume from invalid JSONL row {line_number} in {output_path}"
+                ) from exc
+            if not isinstance(url, str) or not url:
+                raise typer.BadParameter(f"cannot resume from row {line_number} without a valid URL")
+            urls.add(url)
+            index_url = row.get("metadata", {}).get("index_url")
+            if isinstance(index_url, str) and index_url:
+                urls.add(index_url)
+    return frozenset(urls)
+
+
 def _parse_documents(value: str) -> int | None:
     normalized = value.strip().lower()
     if normalized == "all":
diff --git a/datasets/amfv_datasets/scraping/html.py b/datasets/amfv_datasets/scraping/html.py
index 2bd1c968..15805281 100644
--- a/datasets/amfv_datasets/scraping/html.py
+++ b/datasets/amfv_datasets/scraping/html.py
@@ -119,6 +119,8 @@ def _absolutize_links(html_text: str, *, base_url: str) -> str:
     root = lxml_html.fragment_fromstring(html_text, create_parent="div")
     for link in root.xpath(".//a[@href]"):
         link.set("href", urljoin(base_url, link.get("href")))
+    for image in root.xpath(".//img[@src]"):
+        image.set("src", urljoin(base_url, image.get("src")))
     return "".join(lxml_html.tostring(child, encoding="unicode") for child in root)
 
 
diff --git a/datasets/amfv_datasets/scraping/rch.py b/datasets/amfv_datasets/scraping/rch.py
new file mode 100644
index 00000000..db1e1302
--- /dev/null
+++ b/datasets/amfv_datasets/scraping/rch.py
@@ -0,0 +1,398 @@
+"""Scrape RCH clinical practice guidelines into normalized markdown documents.
+
+The Royal Children's Hospital Melbourne (RCH) publishes its clinical practice
+guidelines as structured HTML pages. This module discovers guideline pages from
+the A-Z index and extracts only the primary guideline and reference widgets.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from collections.abc import Collection, Iterable
+from dataclasses import dataclass, replace
+from urllib.parse import unquote, urljoin, urlparse
+
+import httpx
+from lxml import html as lxml_html
+
+from amfv_datasets.scraping.base import (
+    ScrapedDocument,
+    ScrapeError,
+    ScrapeRun,
+    default_client,
+    scrape_listing_documents,
+)
+from amfv_datasets.scraping.html import LinkMode, clean_text, document_title, html_to_markdown
+
+BASE_URL = "https://www.rch.org.au"
+GUIDELINE_INDEX_URL = f"{BASE_URL}/clinicalguide/guideline_index/"
+RCH_DATASET_NAME = "rch-webscrape"
+RCH_DATASET_DISPLAY_NAME = "RCH Clinical Practice Guidelines Webscrape"
+DOCUMENT_DELAY_SECONDS = 10.0
+_LOGGER = logging.getLogger(__name__)
+
+_NON_GUIDELINE_SLUGS = {
+    "CPG_Committee_Calendar",
+    "Fractures",
+    "Immigrant_health_resources",
+}
+
+_GUIDELINE_PATH_RE = re.compile(
+    r"^/clinicalguide/guideline_index/(?P<slug>.+?)/?$",
+    re.IGNORECASE,
+)
+_DIRECT_GUIDELINE_PATHS = {
+    "/persistent_nasal_discharge_rhinosinusitis/.aspx": (
+        "Persistent_nasal_discharge_rhinosinusitis",
+        "/Persistent_nasal_discharge_rhinosinusitis/.aspx",
+    ),
+}
+_SEE_ALIAS_SUFFIX_RE = re.compile(r"\s+\(see\b.*\)$", re.IGNORECASE)
+_LAST_UPDATED_RE = re.compile(r"\bLast updated\s+([^\n]+)", re.IGNORECASE)
+_EMPTY_MARKDOWN_HEADING_RE = re.compile(r"^#{1,6}\s*$", re.MULTILINE)
+_EMPTY_EMPHASIS_RE = re.compile(r"(?<!\*)\*{4}(?!\*)")
+
+
+class RchFetchError(ScrapeError):
+    """Raised when an RCH guideline cannot be discovered or parsed."""
+
+
+@dataclass(frozen=True)
+class RchGuidelineRef:
+    """A clinical practice guideline discovered from the RCH A-Z index."""
+
+    slug: str
+    title: str
+    page_url: str
+    aliases: tuple[str, ...] = ()
+
+
+def guideline_ref_from_url(url: str, *, title: str | None = None) -> RchGuidelineRef:
+    """Parse an RCH guideline URL into a canonical guideline reference.
+
+    Args:
+        url: RCH clinical guideline URL to parse.
+        title: Optional title discovered from the A-Z index (default: None).
+    """
+    parsed = urlparse(url.strip())
+    if parsed.scheme not in {"http", "https"} or parsed.netloc.lower() not in {
+        "www.rch.org.au",
+        "rch.org.au",
+    }:
+        raise RchFetchError(f"Enter an RCH clinical guideline URL from rch.org.au; got {url!r}")
+
+    path = unquote(parsed.path)
+    match = _GUIDELINE_PATH_RE.match(path)
+    if not match:
+        direct_guideline = _DIRECT_GUIDELINE_PATHS.get(path.rstrip("/").casefold())
+        if direct_guideline is not None:
+            slug, canonical_path = direct_guideline
+            return RchGuidelineRef(
+                slug=slug,
+                title=title or slug.replace("_", " "),
+                page_url=f"{BASE_URL}{canonical_path}",
+            )
+        raise RchFetchError(
+            f"Enter a URL like https://www.rch.org.au/clinicalguide/guideline_index/Acute_asthma/; got {url!r}"
+        )
+
+    slug = match.group("slug")
+    if slug.lower().endswith((".pdf", ".doc", ".docx")):
+        raise RchFetchError(f"Enter an RCH HTML clinical guideline URL, not a downloadable file; got {url!r}")
+    trailing_slash = "" if slug.lower().endswith(".aspx") else "/"
+    page_url = f"{BASE_URL}/clinicalguide/guideline_index/{slug}{trailing_slash}"
+    return RchGuidelineRef(slug=slug, title=title or slug.replace("_", " "), page_url=page_url)
+
+
+def list_guidelines(client: httpx.Client) -> list[RchGuidelineRef]:
+    """Return unique guideline references from the RCH A-Z index.
+
+    Args:
+        client: HTTP client used to fetch the guideline index.
+    """
+    response = client.get(GUIDELINE_INDEX_URL)
+    response.raise_for_status()
+    doc = lxml_html.fromstring(response.text)
+    anchors = doc.xpath(
+        "//div[@id='tabnav-letter-blocks' or "
+        "contains(concat(' ', normalize-space(@class), ' '), ' tabnav-letter-blocks ')]//a[@href]"
+    )
+
+    refs_by_url: dict[str, RchGuidelineRef] = {}
+    for anchor in anchors:
+        href = anchor.get("href")
+        title = clean_text(anchor.text_content(), drop_numeric_citations=False)
+        if not href or not title:
+            continue
+        title = _SEE_ALIAS_SUFFIX_RE.sub("", title)
+        try:
+            ref = guideline_ref_from_url(urljoin(BASE_URL, href), title=title)
+        except RchFetchError:
+            continue
+        if ref.slug in _NON_GUIDELINE_SLUGS:
+            continue
+        url_key = ref.page_url.casefold()
+        if existing := refs_by_url.get(url_key):
+            if title != existing.title and title not in existing.aliases:
+                refs_by_url[url_key] = replace(existing, aliases=(*existing.aliases, title))
+            continue
+        refs_by_url[url_key] = ref
+    refs = list(refs_by_url.values())
+    if not refs:
+        raise RchFetchError("Could not find clinical guideline links in the RCH A-Z index")
+    return refs
+
+
+def _content_widgets(html_text: str) -> list[lxml_html.HtmlElement]:
+    doc = lxml_html.fromstring(html_text)
+    widgets = doc.xpath(
+        "//div[@id='rch-primary' or "
+        "contains(concat(' ', normalize-space(@class), ' '), ' rch-primary ')]"
+        "//div[contains(concat(' ', normalize-space(@class), ' '), ' widgetBody ')]"
+    )
+    return [widget for widget in widgets if _is_guideline_widget(widget)]
+
+
+def _is_guideline_widget(widget: lxml_html.HtmlElement) -> bool:
+    if widget.xpath(".//*[self::h2 or self::h3 or self::h4 or self::h5 or self::h6]"):
+        return True
+    text_length = len(clean_text(widget.text_content(), drop_numeric_citations=False))
+    return text_length >= 500 or bool(widget.xpath(".//table")) and text_length >= 200
+
+
+def _normalize_widget(widget: lxml_html.HtmlElement) -> None:
+    for element in widget.xpath(".//script | .//style"):
+        element.drop_tree()
+
+    for nested in widget.xpath(".//strong//strong | .//em//em"):
+        nested.drop_tag()
+
+    for image in widget.xpath(".//img[not(@src) or not(normalize-space(@src))]"):
+        image.drop_tree()
+
+    for element, attribute in (
+        *((link, "href") for link in widget.xpath(".//a[@href]")),
+        *((image, "src") for image in widget.xpath(".//img[@src]")),
+    ):
+        value = element.get(attribute)
+        parsed = urlparse(value)
+        if parsed.netloc.lower() in {"rch.org.au", "www.rch.org.au", "webedit.rch.org.au"}:
+            element.set(attribute, parsed._replace(scheme="https", netloc="www.rch.org.au").geturl())
+
+    for emphasis in widget.xpath(".//strong | .//em | .//b | .//i"):
+        if not clean_text(emphasis.text_content(), drop_numeric_citations=False) and not emphasis.xpath(".//img"):
+            emphasis.drop_tree()
+
+    nested_lists = widget.xpath(".//ul/ul | .//ul/ol | .//ol/ul | .//ol/ol")
+    for nested_list in nested_lists:
+        previous = nested_list.getprevious()
+        if previous is not None and previous.tag.lower() == "li":
+            previous.append(nested_list)
+
+    for heading in widget.xpath(".//*[self::h1 or self::h2 or self::h3 or self::h4 or self::h5 or self::h6]"):
+        for emphasis in heading.xpath(".//strong | .//em | .//b | .//i"):
+            emphasis.drop_tag()
+        if clean_text(heading.text_content(), drop_numeric_citations=False):
+            continue
+        if heading.xpath(".//img"):
+            heading.drop_tag()
+        else:
+            heading.drop_tree()
+
+
+def build_guideline_text(
+    html_text: str,
+    *,
+    link_mode: LinkMode = LinkMode.KEEP,
+    base_url: str = BASE_URL,
+) -> tuple[str, int]:
+    """Extract an RCH guideline page into markdown and a section count.
+
+    Args:
+        html_text: RCH guideline page HTML.
+        link_mode: Whether links are kept as markdown links or stripped to their
+            visible text (default: LinkMode.KEEP).
+        base_url: Document URL used to resolve relative links and fragments
+            (default: BASE_URL).
+    """
+    widgets = _content_widgets(html_text)
+    if not widgets:
+        raise RchFetchError("No readable clinical guideline content found on the RCH page")
+
+    sections: list[str] = []
+    for widget in widgets:
+        _normalize_widget(widget)
+        markdown = html_to_markdown(
+            lxml_html.tostring(widget, encoding="unicode"),
+            link_mode=link_mode,
+            base_url=base_url,
+        )
+        markdown = _EMPTY_MARKDOWN_HEADING_RE.sub("", markdown)
+        markdown = _EMPTY_EMPHASIS_RE.sub("", markdown).strip()
+        if markdown:
+            sections.append(markdown)
+    if not sections:
+        raise RchFetchError("No readable clinical guideline content found on the RCH page")
+    return "\n\n".join(sections), len(sections)
+
+
+def scrape_guideline(
+    client: httpx.Client,
+    ref: RchGuidelineRef,
+    *,
+    link_mode: LinkMode = LinkMode.KEEP,
+) -> ScrapedDocument:
+    """Scrape one RCH guideline into a normalized document.
+
+    Args:
+        client: HTTP client used to fetch the guideline page.
+        ref: RCH guideline reference to scrape.
+        link_mode: Whether links are kept as markdown links or stripped to their
+            visible text (default: LinkMode.KEEP).
+    """
+    response = client.get(ref.page_url)
+    response.raise_for_status()
+    content_url = ref.page_url
+    metadata = {"slug": ref.slug}
+    if ref.aliases:
+        metadata["aliases"] = list(ref.aliases)
+    try:
+        content, section_count = build_guideline_text(
+            response.text,
+            link_mode=link_mode,
+            base_url=content_url,
+        )
+    except RchFetchError:
+        content_url = _linked_content_url(response.text, source_url=ref.page_url)
+        if content_url is None:
+            raise
+        response = client.get(content_url)
+        response.raise_for_status()
+        content, section_count = build_guideline_text(
+            response.text,
+            link_mode=link_mode,
+            base_url=content_url,
+        )
+        metadata["index_url"] = ref.page_url
+    title = document_title(response.text, fallback=ref.title)
+    last_updated_match = _LAST_UPDATED_RE.search(content)
+    if last_updated_match:
+        metadata["last_updated"] = last_updated_match.group(1).strip()
+    return ScrapedDocument(
+        source="rch",
+        external_id=f"rch-{_external_id_slug(ref.slug)}",
+        title=title,
+        url=content_url,
+        content=content,
+        section_count=section_count,
+        metadata=metadata,
+    )
+
+
+def _linked_content_url(html_text: str, *, source_url: str) -> str | None:
+    doc = lxml_html.fromstring(html_text)
+    hrefs = doc.xpath(
+        "//div[@id='rch-primary']//div["
+        "contains(concat(' ', normalize-space(@class), ' '), ' widgetBody ')"
+        "]//a[@href]/@href"
+    )
+    candidates: list[str] = []
+    for href in hrefs:
+        target = urljoin(source_url, href)
+        parsed = urlparse(target)
+        if parsed.scheme not in {"http", "https"} or parsed.netloc.lower() not in {
+            "rch.org.au",
+            "www.rch.org.au",
+        }:
+            continue
+        normalized = target.split("#", maxsplit=1)[0]
+        if normalized != source_url and normalized not in candidates:
+            candidates.append(normalized)
+    return candidates[0] if len(candidates) == 1 else None
+
+
+def _external_id_slug(slug: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "-", slug.lower()).strip("-")
+
+
+def scrape_rch(
+    *,
+    documents: int | None,
+    link_mode: LinkMode = LinkMode.KEEP,
+    url: str | None = None,
+    skip_urls: Collection[str] = (),
+) -> ScrapeRun:
+    """Scrape RCH guidelines from a URL or the A-Z index.
+
+    Args:
+        documents: Number of documents to scrape. Ignored when `url` is set.
+            When unset, every guideline in the A-Z index is scraped (default:
+            None).
+        link_mode: Whether links are kept as markdown links or stripped to their
+            visible text (default: LinkMode.KEEP).
+        url: RCH guideline URL to scrape as a single document (default: None).
+        skip_urls: Canonical document URLs already scraped by a previous run
+            (default: ()).
+    """
+    if url is not None:
+
+        def scrape_url() -> Iterable[ScrapedDocument]:
+            with default_client() as client:
+                yield scrape_guideline(client, guideline_ref_from_url(url), link_mode=link_mode)
+
+        return ScrapeRun(documents=scrape_url(), total=1)
+
+    with default_client() as client:
+        refs = list_guidelines(client)
+    refs = [ref for ref in refs if ref.page_url not in skip_urls]
+    total = len(refs) if documents is None else min(documents, len(refs))
+    return ScrapeRun(
+        total=total,
+        documents=scrape_listing_documents(
+            documents=documents,
+            client_factory=default_client,
+            first_page_items=refs,
+            list_page=lambda _client, _page: (),
+            scrape_item=lambda client, ref: _scrape_or_skip_guideline(client, ref, link_mode=link_mode),
+            document_delay_seconds=DOCUMENT_DELAY_SECONDS,
+        ),
+    )
+
+
+def _scrape_or_skip_guideline(
+    client: httpx.Client,
+    ref: RchGuidelineRef,
+    *,
+    link_mode: LinkMode,
+) -> ScrapedDocument | None:
+    try:
+        return scrape_guideline(client, ref, link_mode=link_mode)
+    except RchFetchError as exc:
+        _LOGGER.warning("Skipping unsupported RCH index entry %s: %s", ref.page_url, exc)
+        return None
+    except httpx.HTTPStatusError as exc:
+        if exc.response.status_code not in {404, 410}:
+            raise
+        _LOGGER.warning(
+            "Skipping stale RCH index entry %s: HTTP %s",
+            ref.page_url,
+            exc.response.status_code,
+        )
+        return None
+
+
+__all__ = [
+    "BASE_URL",
+    "DOCUMENT_DELAY_SECONDS",
+    "GUIDELINE_INDEX_URL",
+    "RCH_DATASET_DISPLAY_NAME",
+    "RCH_DATASET_NAME",
+    "RchFetchError",
+    "RchGuidelineRef",
+    "build_guideline_text",
+    "guideline_ref_from_url",
+    "list_guidelines",
+    "scrape_guideline",
+    "scrape_rch",
+]
diff --git a/datasets/test/test_scraping_base.py b/datasets/test/test_scraping_base.py
index eed3c838..2201e4de 100644
--- a/datasets/test/test_scraping_base.py
+++ b/datasets/test/test_scraping_base.py
@@ -149,3 +149,43 @@ def scrape_item(client: httpx.Client, item: str) -> ScrapedDocument:
 
     assert [document.external_id for document in documents] == ["item-1", "item-2", "item-3"]
     assert delays == [5.0, 5.0]
+
+
+def test_scrape_listing_documents_skips_non_documents_without_consuming_limit(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Skipped listing entries are delayed but do not count as returned documents."""
+    delays: list[float] = []
+
+    class _FakeClient:
+        def __enter__(self) -> "_FakeClient":
+            return self
+
+        def __exit__(self, *args: object) -> None:
+            return None
+
+    def client_factory():
+        return _FakeClient()
+
+    def list_page(client: httpx.Client, page: int) -> list[str]:
+        return ["skip", "item-1", "item-2"] if page == 1 else []
+
+    def scrape_item(client: httpx.Client, item: str) -> ScrapedDocument | None:
+        if item == "skip":
+            return None
+        return ScrapedDocument("test", item, item, f"https://example.org/{item}", "content")
+
+    monkeypatch.setattr(base.time, "sleep", delays.append)
+
+    documents = list(
+        scrape_listing_documents(
+            documents=2,
+            client_factory=client_factory,
+            list_page=list_page,
+            scrape_item=scrape_item,
+            document_delay_seconds=5.0,
+        )
+    )
+
+    assert [document.external_id for document in documents] == ["item-1", "item-2"]
+    assert delays == [5.0, 5.0]
diff --git a/datasets/test/test_scraping_cli.py b/datasets/test/test_scraping_cli.py
index 44f714fa..358f12c6 100644
--- a/datasets/test/test_scraping_cli.py
+++ b/datasets/test/test_scraping_cli.py
@@ -1,7 +1,7 @@
 """Tests for the scraping CLI."""
 
 import json
-from collections.abc import Iterator
+from collections.abc import Collection, Iterator
 from pathlib import Path
 
 import pytest
@@ -11,6 +11,7 @@
 from amfv_datasets.scraping.cli import (
     ScraperSource,
     app,
+    scrape_documents,
     write_huggingface_dataset,
     write_jsonl,
     write_markdown_files,
@@ -194,6 +195,47 @@ def fake_progress(
     assert json.loads(result.stdout.splitlines()[0])["external_id"] == "nice-ng1"
 
 
+def test_cli_run_dispatches_rch_source(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The CLI dispatches RCH source URLs to the RCH scraper."""
+    runner = CliRunner()
+    url = "https://www.rch.org.au/clinicalguide/guideline_index/Acute_asthma/"
+
+    def fake_scrape_rch(
+        *,
+        documents: int | None,
+        link_mode: LinkMode,
+        url: str | None,
+    ) -> ScrapeRun:
+        assert documents == 1
+        assert link_mode is LinkMode.KEEP
+        assert url == "https://www.rch.org.au/clinicalguide/guideline_index/Acute_asthma/"
+        return ScrapeRun([_document(source="rch")], total=1)
+
+    monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_rch", fake_scrape_rch)
+
+    result = runner.invoke(app, ["--source", "rch", "--url", url, "--no-progress"])
+
+    assert result.exit_code == 0
+    assert json.loads(result.stdout.splitlines()[0])["source"] == "rch"
+
+
+def test_scrape_documents_combines_all_sources(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The all source combines documents and totals from every registered scraper."""
+    monkeypatch.setattr(
+        "amfv_datasets.scraping.cli.scrape_nice",
+        lambda **_kwargs: ScrapeRun([_document()], total=1),
+    )
+    monkeypatch.setattr(
+        "amfv_datasets.scraping.cli.scrape_rch",
+        lambda **_kwargs: ScrapeRun([_document(source="rch")], total=1),
+    )
+
+    run = scrape_documents(ScraperSource.ALL, documents=1, link_mode=LinkMode.KEEP)
+
+    assert run.total == 2
+    assert [document.source for document in run] == ["nice", "rch"]
+
+
 def test_cli_run_accepts_all_documents(monkeypatch: pytest.MonkeyPatch) -> None:
     """The CLI accepts --documents all."""
     runner = CliRunner()
@@ -219,10 +261,47 @@ def fake_scrape_documents(
     assert json.loads(result.stdout.splitlines()[0])["external_id"] == "nice-ng1"
 
 
-def _document() -> ScrapedDocument:
+def test_cli_resume_appends_and_skips_existing_urls(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+    """Resume reads completed URLs before scraping and appends new JSONL rows."""
+    output_path = tmp_path / "rch.jsonl"
+    existing = _document()
+    existing_row = existing.__dict__ | {"metadata": {"index_url": "https://example.org/index-entry"}}
+    output_path.write_text(json.dumps(existing_row) + "\n", encoding="utf-8")
+    new_document = ScrapedDocument(
+        source="rch",
+        external_id="rch-new",
+        title="New guideline",
+        url="https://www.rch.org.au/clinicalguide/guideline_index/New/",
+        content="new content",
+    )
+
+    def fake_scrape_documents(
+        source: ScraperSource,
+        *,
+        documents: int | None,
+        link_mode: LinkMode,
+        url: str | None = None,
+        skip_urls: Collection[str] = (),
+    ) -> ScrapeRun:
+        assert source is ScraperSource.RCH
+        assert set(skip_urls) == {existing.url, "https://example.org/index-entry"}
+        return ScrapeRun([new_document], total=1)
+
+    monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_documents", fake_scrape_documents)
+    result = CliRunner().invoke(
+        app,
+        ["--source", "rch", "--output", str(output_path), "--resume", "--no-progress"],
+    )
+
+    assert result.exit_code == 0
+    rows = [json.loads(line) for line in output_path.read_text(encoding="utf-8").splitlines()]
+    assert [row["external_id"] for row in rows] == [existing.external_id, new_document.external_id]
+
+
+def _document(*, source: str = "nice") -> ScrapedDocument:
     return ScrapedDocument(
-        source="nice",
-        external_id="nice-ng1",
+        source=source,
+        external_id=f"{source}-ng1",
         title="Guideline 1",
         url="https://www.nice.org.uk/guidance/ng1",
         content="content",
diff --git a/datasets/test/test_scraping_html.py b/datasets/test/test_scraping_html.py
index d34a8fa3..b228222d 100644
--- a/datasets/test/test_scraping_html.py
+++ b/datasets/test/test_scraping_html.py
@@ -68,3 +68,12 @@ def test_html_to_markdown_can_strip_links() -> None:
     html_text = '<p>Offer <a href="https://example.org">treatment</a>.</p>'
 
     assert html_to_markdown(html_text, link_mode=LinkMode.STRIP) == "Offer treatment."
+
+
+def test_html_to_markdown_absolutizes_images() -> None:
+    """Relative image sources are preserved as absolute markdown image URLs."""
+    html_text = '<img src="/images/flowchart.png" alt="Treatment flowchart">'
+
+    assert html_to_markdown(html_text, base_url="https://example.org/guideline/") == (
+        "![Treatment flowchart](https://example.org/images/flowchart.png)"
+    )
diff --git a/datasets/test/test_scraping_rch.py b/datasets/test/test_scraping_rch.py
new file mode 100644
index 00000000..5b2c4f92
--- /dev/null
+++ b/datasets/test/test_scraping_rch.py
@@ -0,0 +1,252 @@
+"""Tests for RCH clinical guideline scraping helpers."""
+
+import httpx
+import pytest
+
+from amfv_datasets.scraping.html import LinkMode
+from amfv_datasets.scraping.rch import (
+    BASE_URL,
+    GUIDELINE_INDEX_URL,
+    RchGuidelineRef,
+    _scrape_or_skip_guideline,
+    guideline_ref_from_url,
+    list_guidelines,
+    scrape_guideline,
+)
+
+
+def test_list_guidelines_discovers_unique_html_pages() -> None:
+    """The A-Z index yields unique HTML guideline references and nested paths."""
+    html_text = """
+    <html>
+      <div id="tabnav-letter-blocks">
+        <a href="/clinicalguide/guideline_index/Acute_asthma/">Acute asthma</a>
+        <a href="/clinicalguide/guideline_index/Acute_asthma/">Asthma acute (see Acute asthma)</a>
+        <a href="/clinicalguide/guideline_index/fractures/Elbow_Dislocations/">Elbow dislocations</a>
+        <a href="/Persistent_nasal_discharge_rhinosinusitis/.aspx">Nasal discharge</a>
+        <a href="/Persistent_nasal_discharge_rhinosinusitis/.aspx">Rhinosinusitis</a>
+        <a href="/clinicalguide/guideline_index/CPG_Committee_Calendar/">Committee calendar</a>
+        <a href="/clinicalguide/guideline_index/files/package.pdf">PDF package</a>
+      </div>
+      <footer><a href="/clinicalguide/guideline_index/not-a-guideline/">Footer link</a></footer>
+    </html>
+    """
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert str(request.url) == GUIDELINE_INDEX_URL
+        return httpx.Response(200, text=html_text)
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+
+    assert list_guidelines(client) == [
+        RchGuidelineRef(
+            slug="Acute_asthma",
+            title="Acute asthma",
+            page_url=f"{BASE_URL}/clinicalguide/guideline_index/Acute_asthma/",
+            aliases=("Asthma acute",),
+        ),
+        RchGuidelineRef(
+            slug="fractures/Elbow_Dislocations",
+            title="Elbow dislocations",
+            page_url=f"{BASE_URL}/clinicalguide/guideline_index/fractures/Elbow_Dislocations/",
+        ),
+        RchGuidelineRef(
+            slug="Persistent_nasal_discharge_rhinosinusitis",
+            title="Nasal discharge",
+            page_url=f"{BASE_URL}/Persistent_nasal_discharge_rhinosinusitis/.aspx",
+            aliases=("Rhinosinusitis",),
+        ),
+    ]
+
+
+def test_guideline_ref_from_url_normalizes_host_and_query() -> None:
+    """RCH guideline URLs are normalized to canonical HTTPS page URLs."""
+    assert guideline_ref_from_url(
+        "http://rch.org.au/clinicalguide/guideline_index/Acute_asthma/?print=yes#management"
+    ) == RchGuidelineRef(
+        slug="Acute_asthma",
+        title="Acute asthma",
+        page_url="https://www.rch.org.au/clinicalguide/guideline_index/Acute_asthma/",
+    )
+
+
+def test_guideline_ref_from_url_preserves_legacy_aspx_route() -> None:
+    """Legacy ASPX guideline paths do not receive a breaking trailing slash."""
+    assert (
+        guideline_ref_from_url("https://www.rch.org.au/clinicalguide/guideline_index/Gastrostomy.aspx/").page_url
+        == "https://www.rch.org.au/clinicalguide/guideline_index/Gastrostomy.aspx"
+    )
+
+
+def test_guideline_ref_from_url_accepts_direct_rhinosinusitis_guideline() -> None:
+    """The clinical A-Z index's direct rhinosinusitis route remains discoverable."""
+    assert guideline_ref_from_url(
+        "https://rch.org.au/Persistent_nasal_discharge_rhinosinusitis/.aspx?print=yes"
+    ) == RchGuidelineRef(
+        slug="Persistent_nasal_discharge_rhinosinusitis",
+        title="Persistent nasal discharge rhinosinusitis",
+        page_url=f"{BASE_URL}/Persistent_nasal_discharge_rhinosinusitis/.aspx",
+    )
+
+
+def test_scrape_guideline_extracts_primary_widgets_as_markdown() -> None:
+    """Guideline content, references, metadata, lists, tables, and images are preserved."""
+    html_text = """
+    <html>
+      <h1>Acute asthma</h1>
+      <div id="rch-primary" class="col-md-10">
+        <div class="widgetBody"><img alt="PIC Endorsed" src="/pic-logo.png"></div>
+        <div class="widgetBody">
+          <h2>Key points</h2>
+          <p><strong>Treat urgently.</strong><strong></strong></p>
+          <ul>
+            <li>Assess severity:</li>
+            <ul><li>Check breathing.</li><li>Check activity.</li></ul>
+          </ul>
+          <h3><strong><strong>Management</strong> details</strong></h3>
+          <p>See the <a href="/clinicalguide/guideline_index/Anaphylaxis/">anaphylaxis guideline</a>.</p>
+          <p>Jump to <a href="#management">management</a>.</p>
+          <table><tr><th>Severity</th><th>Action</th></tr><tr><td>Severe</td><td>Escalate</td></tr></table>
+          <h4><br></h4>
+          <p><img src="/uploadedImages/asthma-flowchart.png" alt="Asthma flowchart"></p>
+          <p><img src="http://webedit.rch.org.au/uploadedImages/legacy.png" alt="Legacy image"></p>
+          <p><img alt="Missing image source"></p>
+          <p>Last updated July 2025</p>
+          <script>window.rchWidget = {"tracking": true};</script>
+        </div>
+        <div class="widgetBody"><h2>Reference List</h2><ol><li>Reference one.</li></ol></div>
+      </div>
+      <footer><h2>Contact us</h2><p>Footer content.</p></footer>
+    </html>
+    """
+    ref = RchGuidelineRef(
+        slug="Acute_asthma",
+        title="Acute asthma listing title",
+        page_url="https://www.rch.org.au/clinicalguide/guideline_index/Acute_asthma/",
+        aliases=("Asthma acute", "Wheeze"),
+    )
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert str(request.url) == ref.page_url
+        return httpx.Response(200, text=html_text)
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+
+    document = scrape_guideline(client, ref)
+
+    assert document.source == "rch"
+    assert document.external_id == "rch-acute-asthma"
+    assert document.title == "Acute asthma"
+    assert document.section_count == 2
+    assert document.metadata == {
+        "slug": "Acute_asthma",
+        "aliases": ["Asthma acute", "Wheeze"],
+        "last_updated": "July 2025",
+    }
+    assert "- Assess severity:\n  - Check breathing.\n  - Check activity." in document.content
+    assert "### Management details" in document.content
+    assert (
+        "[anaphylaxis guideline](https://www.rch.org.au/clinicalguide/guideline_index/Anaphylaxis/)" in document.content
+    )
+    assert (
+        "[management](https://www.rch.org.au/clinicalguide/guideline_index/Acute_asthma/#management)"
+        in document.content
+    )
+    assert "| Severity | Action |" in document.content
+    assert "![Asthma flowchart](https://www.rch.org.au/uploadedImages/asthma-flowchart.png)" in document.content
+    assert "![Legacy image](https://www.rch.org.au/uploadedImages/legacy.png)" in document.content
+    assert "Missing image source" not in document.content
+    assert "## Reference List" in document.content
+    assert "****" not in document.content
+    assert "PIC Endorsed" not in document.content
+    assert "Footer content" not in document.content
+    assert "rchWidget" not in document.content
+
+
+def test_scrape_guideline_keeps_substantial_headingless_legacy_content() -> None:
+    """Legacy clinical tables without section headings remain valid guideline content."""
+    clinical_rows = "".join(f"<tr><td>Disease {index}</td><td>Dose {index}</td></tr>" for index in range(10))
+    html_text = f"""
+    <html><h1>Empiric treatment</h1><div id="rch-primary"><div class="widgetBody">
+      <p>Start empiric treatment based on clinical features and local antimicrobial susceptibility patterns.</p>
+      <table><tr><th>Disease</th><th>Treatment</th></tr>{clinical_rows}</table>
+    </div></div></html>
+    """
+    ref = RchGuidelineRef("Empiric_treatment", "Empiric treatment", f"{BASE_URL}/guideline/empiric")
+
+    with httpx.Client(transport=httpx.MockTransport(lambda _request: httpx.Response(200, text=html_text))) as client:
+        document = scrape_guideline(client, ref)
+
+    assert "| Disease | Treatment |" in document.content
+    assert "local antimicrobial susceptibility patterns" in document.content
+
+
+def test_scrape_guideline_resolves_same_site_link_wrapper() -> None:
+    """Link-only A-Z entries resolve to their substantial same-site content pages."""
+    index_url = f"{BASE_URL}/clinicalguide/guideline_index/IV_Immunoglobulin/"
+    target_url = f"{BASE_URL}/bloodtrans/about_blood_products/Intravenous_Immunoglobulin_Guideline"
+    wrapper_html = f"""
+    <html><h1>IV Immunoglobulin</h1><div id="rch-primary"><div class="widgetBody">
+      <p><a href="{target_url}">Go to the IVIg guideline</a></p>
+    </div></div></html>
+    """
+    target_html = """
+    <html><h1>Intravenous immunoglobulin</h1><div id="rch-primary"><div class="widgetBody">
+      <h2>Administration</h2><p>Monitor the patient throughout the infusion.</p>
+    </div></div></html>
+    """
+    ref = RchGuidelineRef("IV_Immunoglobulin", "IV Immunoglobulin", index_url)
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        pages = {index_url: wrapper_html, target_url: target_html}
+        return httpx.Response(200, text=pages[str(request.url)])
+
+    with httpx.Client(transport=httpx.MockTransport(handler)) as client:
+        document = scrape_guideline(client, ref)
+
+    assert document.url == target_url
+    assert document.title == "Intravenous immunoglobulin"
+    assert document.metadata == {"slug": "IV_Immunoglobulin", "index_url": index_url}
+    assert "Monitor the patient throughout the infusion." in document.content
+
+
+def test_scrape_or_skip_guideline_logs_non_guideline_index_entry(caplog: pytest.LogCaptureFixture) -> None:
+    """Resource pages discovered in the guideline index are reported and skipped."""
+    ref = RchGuidelineRef(
+        slug="Immigrant_health_resources",
+        title="Immigrant health resources",
+        page_url="https://www.rch.org.au/clinicalguide/guideline_index/Immigrant_health_resources/",
+    )
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert str(request.url) == ref.page_url
+        html_text = "<html><div id='rch-primary'><div class='widgetBody'>Resource link</div></div></html>"
+        return httpx.Response(200, text=html_text)
+
+    with httpx.Client(transport=httpx.MockTransport(handler)) as client:
+        assert _scrape_or_skip_guideline(client, ref, link_mode=LinkMode.KEEP) is None
+
+    assert "Skipping unsupported RCH index entry" in caplog.text
+
+
+@pytest.mark.parametrize("status_code", [404, 410])
+def test_scrape_or_skip_guideline_logs_stale_index_entry(
+    status_code: int,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Permanently missing pages in the live index are reported and skipped."""
+    ref = RchGuidelineRef("stale", "Stale", f"{BASE_URL}/clinicalguide/guideline_index/stale/")
+
+    with httpx.Client(transport=httpx.MockTransport(lambda _request: httpx.Response(status_code))) as client:
+        assert _scrape_or_skip_guideline(client, ref, link_mode=LinkMode.KEEP) is None
+
+    assert f"HTTP {status_code}" in caplog.text
+
+
+def test_scrape_or_skip_guideline_raises_transient_http_error() -> None:
+    """Server errors still abort so incomplete runs are not silently accepted."""
+    ref = RchGuidelineRef("error", "Error", f"{BASE_URL}/clinicalguide/guideline_index/error/")
+
+    with httpx.Client(transport=httpx.MockTransport(lambda _request: httpx.Response(503))) as client:
+        with pytest.raises(httpx.HTTPStatusError):
+            _scrape_or_skip_guideline(client, ref, link_mode=LinkMode.KEEP)

From b55d196af48bf57905560acc792c774102a6dbff Mon Sep 17 00:00:00 2001
From: Zander Giuffrida <zndr27@gmail.com>
Date: Sun, 26 Jul 2026 00:05:17 -0700
Subject: [PATCH 06/15] allow scrape_item to skip a listed item

Taken verbatim from #10 so the two merge cleanly in either order. A listing
entry is not always a document: PMC holds some open-access records as metadata
only, with no full text deposited. Returning None skips the entry without
consuming the --documents budget, and the delay gate moves from scraped to
attempted so skipped items still rate-limit.

Drop this commit if #10 lands first.
---
 datasets/amfv_datasets/scraping/base.py | 14 ++++++---
 datasets/test/test_scraping_base.py     | 40 +++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/datasets/amfv_datasets/scraping/base.py b/datasets/amfv_datasets/scraping/base.py
index 0708ee87..3651cdb7 100644
--- a/datasets/amfv_datasets/scraping/base.py
+++ b/datasets/amfv_datasets/scraping/base.py
@@ -74,7 +74,7 @@ def scrape_listing_documents[ListingItemT](
     documents: int | None,
     client_factory: Callable[[], AbstractContextManager[httpx.Client]],
     list_page: Callable[[httpx.Client, int], Iterable[ListingItemT]],
-    scrape_item: Callable[[httpx.Client, ListingItemT], ScrapedDocument],
+    scrape_item: Callable[[httpx.Client, ListingItemT], ScrapedDocument | None],
     document_delay_seconds: float = 5.0,
     first_page_items: Iterable[ListingItemT] | None = None,
 ) -> Iterable[ScrapedDocument]:
@@ -85,7 +85,8 @@ def scrape_listing_documents[ListingItemT](
             fetched until a page returns no items (default: None).
         client_factory: Factory returning a context-managed HTTP client.
         list_page: Function that lists source-specific items for a page.
-        scrape_item: Function that scrapes one listed item into a document.
+        scrape_item: Function that scrapes one listed item into a document. It
+            may return None to skip a discovered item that is not a document.
         document_delay_seconds: Delay before scraping each document after the
             first one (default: 5.0).
         first_page_items: Already-fetched first listing page items. When set,
@@ -99,6 +100,7 @@ def scrape_listing_documents[ListingItemT](
     with client_factory() as client:
         page = 1
         scraped = 0
+        attempted = 0
         page_items = list(first_page_items) if first_page_items is not None else None
         while documents is None or scraped < documents:
             if page_items is None:
@@ -111,9 +113,13 @@ def scrape_listing_documents[ListingItemT](
             for item in items:
                 if documents is not None and scraped >= documents:
                     break
-                if scraped and document_delay_seconds:
+                if attempted and document_delay_seconds:
                     time.sleep(document_delay_seconds)
-                yield scrape_item(client, item)
+                document = scrape_item(client, item)
+                attempted += 1
+                if document is None:
+                    continue
+                yield document
                 scraped += 1
             page += 1
 
diff --git a/datasets/test/test_scraping_base.py b/datasets/test/test_scraping_base.py
index eed3c838..2201e4de 100644
--- a/datasets/test/test_scraping_base.py
+++ b/datasets/test/test_scraping_base.py
@@ -149,3 +149,43 @@ def scrape_item(client: httpx.Client, item: str) -> ScrapedDocument:
 
     assert [document.external_id for document in documents] == ["item-1", "item-2", "item-3"]
     assert delays == [5.0, 5.0]
+
+
+def test_scrape_listing_documents_skips_non_documents_without_consuming_limit(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Skipped listing entries are delayed but do not count as returned documents."""
+    delays: list[float] = []
+
+    class _FakeClient:
+        def __enter__(self) -> "_FakeClient":
+            return self
+
+        def __exit__(self, *args: object) -> None:
+            return None
+
+    def client_factory():
+        return _FakeClient()
+
+    def list_page(client: httpx.Client, page: int) -> list[str]:
+        return ["skip", "item-1", "item-2"] if page == 1 else []
+
+    def scrape_item(client: httpx.Client, item: str) -> ScrapedDocument | None:
+        if item == "skip":
+            return None
+        return ScrapedDocument("test", item, item, f"https://example.org/{item}", "content")
+
+    monkeypatch.setattr(base.time, "sleep", delays.append)
+
+    documents = list(
+        scrape_listing_documents(
+            documents=2,
+            client_factory=client_factory,
+            list_page=list_page,
+            scrape_item=scrape_item,
+            document_delay_seconds=5.0,
+        )
+    )
+
+    assert [document.external_id for document in documents] == ["item-1", "item-2"]
+    assert delays == [5.0, 5.0]

From 65273392ed20682323702dff2c8d78ca4fcf8f00 Mon Sep 17 00:00:00 2001
From: Zander Giuffrida <zndr27@gmail.com>
Date: Sun, 26 Jul 2026 14:53:55 -0700
Subject: [PATCH 07/15] add PubMed guideline scraper

Scrapes the intersection of PubMed's Guideline publication type with the PMC
Open Access subset, restricted to English: about 3,000 guidelines. That is the
only slice where the full text is both retrievable and openly licensed; the
other 40k Guideline records expose an abstract only, under publisher copyright.

Two scoping choices, both measured rather than assumed:

`Guideline[pt]` rather than `Practice Guideline[pt]`. The former is a strict
superset and the 352 records it adds are clinical, not administrative (the 2025
Korean CPR guidelines and similar), so the narrower tag would drop 11% of the
corpus for nothing.

`English[la]`, which drops 185 records. Every other source in this package is
already English-only as a side effect of its entry URL: CPS is a bilingual site
scraped through its /en/ routes, WHO publishes in six languages and is scraped
through its English listing. PubMed's API returns every language, so the filter
has to be explicit to match. The excluded records are largely French CMAJ
translations of guidelines already in the corpus.

Discovery and extraction use NCBI's E-utilities rather than the rendered pages.
esearch paginates by numeric offset, so the page number maps straight onto
retstart and an empty page past the end terminates the run. esummary resolves a
whole batch of PMIDs to PMCIDs and citation metadata in one request. efetch
returns JATS XML carrying body, section structure and license.

JATS is close enough to HTML that renaming tags and reusing html_to_markdown is
cheaper and less error-prone than a second serializer: table-wrap already
contains genuine XHTML tables, and inline markup maps one to one.

Licensing is recorded per document rather than claimed for the source, because
the Open Access Subset is not uniformly Creative Commons licensed. Censused
over all 2,999 English records present on 2026-07-29:

  CC BY          44.5%    publisher terms, no CC license   11.7%
  CC BY-NC       22.1%    of which: Elsevier COVID grant, no
  CC BY-NC-ND    18.4%              <license> element at all (112),
  CC BY-NC-SA     2.1%              PMC OA "unrestricted re-use"
  CC0             1.3%

By what that permits: 45.8% unrestricted for derivative works, 24.1%
non-commercial only, 18.4% asserting NoDerivatives, 11.7% needing a
case-by-case reading. Presence in the subset is not itself a grant to
redistribute: 112 records carry only a copyright line such as "(c)
Springer-Verlag Tokyo 2007", and Elsevier's pandemic-era deposits grant free
access while still reserving all rights.

The license name is parsed from the Creative Commons URL rather than the
license-type attribute, which the corpus spells 18 different ways. The
copyright statement is captured separately because it is a sibling of
<license>, not a child, and it holds the reservation of rights.

Figures and supplementary files are recorded in metadata rather than linked.
Unlike the HTML sources in #9 and #10, which absolutize a real <img src>, JATS
carries only a bare filename; the served URL inserts a CDN shard and content
hash that appear nowhere in the API response, so a constructed link 404s.
Supplementary blocks are pointers too: across 40 sampled guidelines every one
referenced an external .docx or .tif rather than inline content, totalling
0.18% of body text. Recording name, label and caption keeps the evidence
tables findable without re-scraping.

E-utilities calls retry with backoff on 429 and 5xx. One document makes up to
two calls back to back and the rate limit is per source address, so NCBI does
answer with 429 in practice; without a retry that propagates past the skip
handler and kills the whole run.

external_id prefers the PMID from the record itself, so an article reached
from a PMC URL gets the same identifier as one reached from the listing.

Records PMC holds without a deposited body, 0.6% of the corpus, are logged
and skipped rather than aborting the run.
---
 datasets/amfv_datasets/scraping/__init__.py |  18 +
 datasets/amfv_datasets/scraping/cli.py      |   6 +-
 datasets/amfv_datasets/scraping/pubmed.py   | 723 ++++++++++++++++++++
 datasets/test/test_scraping_pubmed.py       | 478 +++++++++++++
 4 files changed, 1224 insertions(+), 1 deletion(-)
 create mode 100644 datasets/amfv_datasets/scraping/pubmed.py
 create mode 100644 datasets/test/test_scraping_pubmed.py

diff --git a/datasets/amfv_datasets/scraping/__init__.py b/datasets/amfv_datasets/scraping/__init__.py
index cd7cdb06..d5630df7 100644
--- a/datasets/amfv_datasets/scraping/__init__.py
+++ b/datasets/amfv_datasets/scraping/__init__.py
@@ -27,6 +27,16 @@
     scrape_guideline,
     scrape_nice,
 )
+from amfv_datasets.scraping.pubmed import (
+    PubMedArticleRef,
+    PubMedFetchError,
+    PubMedListingPage,
+    build_pubmed_article_text,
+    list_pubmed_guidelines,
+    pubmed_ref_from_url,
+    scrape_pubmed,
+    scrape_pubmed_article,
+)
 
 __all__ = [
     "GuidanceRef",
@@ -34,6 +44,9 @@
     "LinkMode",
     "NiceFetchError",
     "OutputFormat",
+    "PubMedArticleRef",
+    "PubMedFetchError",
+    "PubMedListingPage",
     "ScrapeError",
     "ScrapeRun",
     "ScrapedDocument",
@@ -41,6 +54,7 @@
     "USER_AGENT",
     "absolute_unique_urls",
     "build_guideline_text",
+    "build_pubmed_article_text",
     "clean_text",
     "document_title",
     "default_client",
@@ -48,7 +62,11 @@
     "guidance_ref_from_url",
     "html_to_markdown",
     "list_published_guidance",
+    "list_pubmed_guidelines",
+    "pubmed_ref_from_url",
     "scrape_guideline",
     "scrape_listing_documents",
     "scrape_nice",
+    "scrape_pubmed",
+    "scrape_pubmed_article",
 ]
diff --git a/datasets/amfv_datasets/scraping/cli.py b/datasets/amfv_datasets/scraping/cli.py
index e8aece34..32cafb16 100644
--- a/datasets/amfv_datasets/scraping/cli.py
+++ b/datasets/amfv_datasets/scraping/cli.py
@@ -27,6 +27,7 @@
 from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun
 from amfv_datasets.scraping.html import LinkMode
 from amfv_datasets.scraping.nice import scrape_nice
+from amfv_datasets.scraping.pubmed import scrape_pubmed
 
 
 class ScraperSource(StrEnum):
@@ -34,6 +35,7 @@ class ScraperSource(StrEnum):
 
     ALL = "all"
     NICE = "nice"
+    PUBMED = "pubmed"
 
 
 class OutputFormat(StrEnum):
@@ -72,6 +74,8 @@ def scrape_documents(
         match selected_source:
             case ScraperSource.NICE:
                 return scrape_nice(documents=documents, link_mode=link_mode, url=url)
+            case ScraperSource.PUBMED:
+                return scrape_pubmed(documents=documents, link_mode=link_mode, url=url)
             case ScraperSource.ALL:
                 raise AssertionError("expanded source cannot be all")
     raise AssertionError(f"unsupported source: {source}")
@@ -125,7 +129,7 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path
 
 def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]:
     if source is ScraperSource.ALL:
-        return (ScraperSource.NICE,)
+        return (ScraperSource.NICE, ScraperSource.PUBMED)
     return (source,)
 
 
diff --git a/datasets/amfv_datasets/scraping/pubmed.py b/datasets/amfv_datasets/scraping/pubmed.py
new file mode 100644
index 00000000..5a1f411b
--- /dev/null
+++ b/datasets/amfv_datasets/scraping/pubmed.py
@@ -0,0 +1,723 @@
+"""Scrape PubMed clinical practice guidelines into normalized markdown documents.
+
+PubMed indexes roughly 43k articles tagged with the `Guideline` publication
+type, but most of them expose an abstract only and stay under publisher
+copyright. This module deliberately scrapes the intersection with the PMC Open
+Access subset instead:
+
+    Guideline[pt] AND pubmed pmc open access[filter] AND English[la]
+
+That is about 3,000 guidelines, growing by a handful a week, and it is the only
+slice where the full text is both retrievable and openly licensed.
+
+Licensing is recorded per document rather than claimed for the source, because
+the Open Access Subset is not uniformly Creative Commons licensed. Censused
+over all 2,999 records present on 2026-07-29: CC BY 44.5%, CC BY-NC 22.1%,
+CC BY-NC-ND 18.4%, CC BY-NC-SA 2.1%, CC0 1.3%, and 11.7% carrying publisher
+terms instead. By what that permits, 45.8% is unrestricted for derivative
+works, 24.1% is non-commercial only, 18.4% asserts NoDerivatives, and 11.7%
+has to be read case by case.
+
+Discovery and extraction both go through NCBI's E-utilities rather than the
+rendered pubmed.ncbi.nlm.nih.gov pages. `esearch` paginates by numeric offset and
+reports the result total, `esummary` resolves a whole batch of PMIDs to PMCIDs
+and citation metadata in one request, and `efetch` returns JATS XML carrying the
+article body, section structure and license. All three are documented, versioned
+contracts that do not break when the website is restyled.
+
+Attribution:
+Meditron's guideline scrapers (epfLLM/meditron, gap-replay/guidelines) cover a
+dozen sources but have no PubMed scraper, so this module has no upstream port to
+follow. It follows the conventions of `nice.py` in this package instead.
+Source license: Apache License 2.0.
+"""
+
+from __future__ import annotations
+
+import copy
+import logging
+import os
+import re
+import time
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import urlparse
+
+import httpx
+from lxml import etree
+
+from amfv_datasets.scraping.base import (
+    ScrapedDocument,
+    ScrapeError,
+    ScrapeRun,
+    default_client,
+    scrape_listing_documents,
+)
+from amfv_datasets.scraping.html import LinkMode, clean_text, html_to_markdown
+
+BASE_URL = "https://pubmed.ncbi.nlm.nih.gov"
+EUTILS_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"
+PMC_ARTICLE_URL = "https://www.ncbi.nlm.nih.gov/pmc/articles"
+# `Guideline` is a strict superset of `Practice Guideline`: searching for either
+# returns exactly the `Guideline` count. The 352 extra records are clinical, not
+# administrative — the 2025 Korean CPR guidelines and similar — so scoping to
+# `Practice Guideline` would drop 352 records (11%) for nothing.
+#
+# `English[la]` drops 185 records. Most other sources in this package are
+# English-only as a side effect of the URL they start from (CPS is a bilingual
+# site scraped through its `/en/` routes; WHO publishes in six languages and is
+# scraped through its English listing). PubMed's API returns every language, so
+# the filter is explicit here to match. The excluded records are largely French
+# CMAJ translations of English guidelines already in the corpus.
+SEARCH_TERM = "Guideline[pt] AND pubmed pmc open access[filter] AND English[la]"
+PUBMED_DATASET_NAME = "pubmed-webscrape"
+PUBMED_DATASET_DISPLAY_NAME = "PubMed Webscrape"
+NCBI_TOOL = "amfv"
+# NCBI documents 3 requests/second without an API key and 10 with one, so unlike
+# a scraped website this delay is a published allowance rather than a guess. Do
+# not raise it to match the other scrapers "for consistency" — that turns a
+# 20-minute run into a 4-hour one for no benefit.
+DOCUMENT_DELAY_SECONDS = 0.4
+LISTING_PAGE_SIZE = 200
+# Enough of a non-CC license statement to tell "unrestricted re-use" apart from
+# "all rights reserved" without carrying a wall of boilerplate per document.
+_LICENSE_STATEMENT_CHARS = 400
+_FIGURE_CAPTION_CHARS = 300
+# NCBI answers a burst with 429 and is transiently unavailable often enough that
+# a multi-thousand document run needs to ride both out.
+_MAX_RETRIES = 4
+_RETRY_BACKOFF_SECONDS = 1.0
+_RETRYABLE_STATUS = frozenset({429, 500, 502, 503, 504})
+
+logger = logging.getLogger(__name__)
+
+_PUBMED_PATH_RE = re.compile(r"^/(?P<pmid>\d+)/?$")
+_PMC_PATH_RE = re.compile(r"^/pmc/articles/(?P<pmcid>PMC\d+)/?$", re.IGNORECASE)
+_CC_URL_RE = re.compile(r"https?://creativecommons\.org/\S+?(?=[\s\"<)]|$)")
+# Unwrapping `xref` and dropping figures leaves the surrounding spacing behind,
+# as in "(PRISMA flow diagram in Supplementary Fig. S1) ." — close it back up.
+_ORPHAN_SPACE_RE = re.compile(r" +([,.;:)\]])")
+_XLINK_HREF = "{http://www.w3.org/1999/xlink}href"
+_ALI_LICENSE_REF = "{http://www.niso.org/schemas/ali/1.0/}license_ref"
+
+# JATS elements whose entire subtree is dropped. Reference lists and footnote
+# groups are citation apparatus rather than clinical content. Figure payloads
+# cannot be linked from the API response at all (see `_article_figures`), so the
+# `<graphic>` goes but its `<label>` and `<caption>` stay in the text, and the
+# filename is recorded in `metadata.figures` for a later pass to resolve.
+_DROPPED_TAGS = (
+    "ref-list",
+    "fn-group",
+    "supplementary-material",
+    "table-wrap-foot",
+    "graphic",
+    "inline-graphic",
+    "media",
+    "object-id",
+    "alt-text",
+)
+# Elements unwrapped to their text. `xref` is a citation/figure pointer whose
+# rendered text is noise like "[12]", which `html_to_markdown` then strips.
+_UNWRAPPED_TAGS = ("xref", "styled-content", "named-content")
+_RENAMED_TAGS = {
+    "sec": "div",
+    "italic": "em",
+    "bold": "strong",
+    "underline": "u",
+    "monospace": "code",
+    "list-item": "li",
+    "table-wrap": "div",
+    "disp-quote": "blockquote",
+    "break": "br",
+    "label": "strong",
+}
+
+
+class PubMedFetchError(ScrapeError):
+    """Raised when a guideline cannot be sourced from PubMed or PMC."""
+
+
+@dataclass(frozen=True)
+class PubMedArticleRef:
+    """A PubMed guideline reference resolved from the search listing."""
+
+    pmid: str = ""
+    pmcid: str = ""
+    title: str = ""
+    journal: str = ""
+    pub_date: str = ""
+    doi: str = ""
+
+    @property
+    def page_url(self) -> str:
+        """Canonical PubMed record URL, falling back to the PMC article URL."""
+        return f"{BASE_URL}/{self.pmid}/" if self.pmid else self.pmc_url
+
+    @property
+    def pmc_url(self) -> str:
+        """PMC full-text article URL."""
+        return f"{PMC_ARTICLE_URL}/{self.pmcid}/"
+
+    @property
+    def slug(self) -> str:
+        """Stable identifier used for `external_id`."""
+        return self.pmid or self.pmcid
+
+
+@dataclass(frozen=True)
+class PubMedListingPage:
+    """One page of PubMed guideline search results."""
+
+    refs: list[PubMedArticleRef]
+    total: int | None
+
+
+def pubmed_ref_from_url(url: str) -> PubMedArticleRef:
+    """Parse a PubMed or PMC article URL into a guideline reference.
+
+    Only one of the two identifiers is known from a URL; the other is resolved
+    against E-utilities when the article is scraped.
+
+    Args:
+        url: PubMed or PMC article URL to parse.
+    """
+    parsed = urlparse(url.strip())
+    netloc = parsed.netloc.lower()
+    if parsed.scheme not in {"http", "https"}:
+        raise PubMedFetchError(f"Enter an http(s) PubMed or PMC URL; got {url!r}")
+
+    if netloc in {"pubmed.ncbi.nlm.nih.gov", "www.pubmed.ncbi.nlm.nih.gov"}:
+        match = _PUBMED_PATH_RE.match(parsed.path)
+        if match:
+            return PubMedArticleRef(pmid=match.group("pmid"))
+    elif netloc in {"www.ncbi.nlm.nih.gov", "ncbi.nlm.nih.gov", "pmc.ncbi.nlm.nih.gov"}:
+        match = _PMC_PATH_RE.match(parsed.path)
+        if match:
+            return PubMedArticleRef(pmcid=match.group("pmcid").upper())
+
+    raise PubMedFetchError(f"Enter a URL like {BASE_URL}/42476581/ or {PMC_ARTICLE_URL}/PMC13384732/; got {url!r}")
+
+
+def _policy_params() -> dict[str, str]:
+    """Identify the client per NCBI's E-utilities usage policy.
+
+    NCBI asks every request to carry a `tool` name and contact `email`, and
+    raises the rate limit from 3 to 10 requests/second when an `api_key` is
+    supplied. Both optional values come from the environment so that no personal
+    address is committed to the repository.
+    """
+    params = {"tool": NCBI_TOOL}
+    for name, variable in (("email", "NCBI_EMAIL"), ("api_key", "NCBI_API_KEY")):
+        value = os.environ.get(variable)
+        if value:
+            params[name] = value
+    return params
+
+
+def _eutils_get(client: httpx.Client, endpoint: str, params: dict[str, Any]) -> httpx.Response:
+    """Call one E-utilities endpoint, retrying when NCBI asks us to slow down.
+
+    `DOCUMENT_DELAY_SECONDS` paces one document against the next, but a single
+    document still makes up to two calls back to back, and the published 3/s
+    limit is per source address rather than per process. NCBI answers a burst
+    with 429. Without a retry that propagates out of `_scrape_or_skip`, which
+    only swallows `PubMedFetchError`, and kills a 3,000-document run partway
+    through.
+    """
+    url = f"{EUTILS_URL}/{endpoint}"
+    merged = {**params, **_policy_params()}
+    for attempt in range(_MAX_RETRIES):
+        response = client.get(url, params=merged)
+        if response.status_code in _RETRYABLE_STATUS and attempt < _MAX_RETRIES - 1:
+            delay = _RETRY_BACKOFF_SECONDS * 2**attempt
+            logger.warning("NCBI returned %s for %s; retrying in %.1fs", response.status_code, endpoint, delay)
+            time.sleep(delay)
+            continue
+        response.raise_for_status()
+        return response
+    raise PubMedFetchError(f"E-utilities {endpoint} did not succeed after {_MAX_RETRIES} attempts")
+
+
+def _summarize(client: httpx.Client, pmids: list[str]) -> list[PubMedArticleRef]:
+    """Resolve a batch of PMIDs to references carrying PMCIDs and citation data."""
+    payload = _eutils_get(
+        client,
+        "esummary.fcgi",
+        {"db": "pubmed", "id": ",".join(pmids), "retmode": "json"},
+    ).json()
+    result = payload.get("result", {})
+
+    refs: list[PubMedArticleRef] = []
+    for pmid in result.get("uids", []):
+        record = result.get(pmid, {})
+        article_ids = {entry.get("idtype"): entry.get("value") for entry in record.get("articleids", [])}
+        pmcid = article_ids.get("pmc", "")
+        if not pmcid:
+            # The open-access filter should guarantee a PMC copy, so a missing
+            # one means the record changed between search and summary.
+            logger.warning("Skipping PubMed record %s with no PMC identifier", pmid)
+            continue
+        refs.append(
+            PubMedArticleRef(
+                pmid=pmid,
+                pmcid=pmcid,
+                title=clean_text(record.get("title", ""), drop_numeric_citations=False),
+                journal=record.get("fulljournalname", ""),
+                pub_date=record.get("pubdate", ""),
+                doi=article_ids.get("doi", ""),
+            )
+        )
+    return refs
+
+
+def list_pubmed_guidelines(client: httpx.Client, page: int = 1) -> PubMedListingPage:
+    """List one page of open-access PubMed practice guidelines.
+
+    `esearch` paginates by numeric offset, so the page number maps directly onto
+    `retstart` and a page past the end returns an empty list rather than an
+    error. That is what terminates the scrape loop.
+
+    Args:
+        client: HTTP client used to call E-utilities.
+        page: 1-based listing page number (default: 1).
+    """
+    payload = _eutils_get(
+        client,
+        "esearch.fcgi",
+        {
+            "db": "pubmed",
+            "term": SEARCH_TERM,
+            "retstart": (page - 1) * LISTING_PAGE_SIZE,
+            "retmax": LISTING_PAGE_SIZE,
+            "retmode": "json",
+        },
+    ).json()
+    result = payload.get("esearchresult", {})
+    if "ERROR" in result:
+        raise PubMedFetchError(f"PubMed search error: {result['ERROR']}")
+
+    raw_total = str(result.get("count", ""))
+    total = int(raw_total) if raw_total.isdigit() else None
+    pmids = [str(pmid) for pmid in result.get("idlist", [])]
+    if not pmids:
+        return PubMedListingPage(refs=[], total=total)
+    return PubMedListingPage(refs=_summarize(client, pmids), total=total)
+
+
+def _fetch_pmc_article(client: httpx.Client, pmcid: str) -> etree._Element:
+    """Fetch one PMC record and return its `<article>` element."""
+    response = _eutils_get(
+        client,
+        "efetch.fcgi",
+        {"db": "pmc", "id": pmcid.upper().removeprefix("PMC"), "retmode": "xml"},
+    )
+    try:
+        root = etree.fromstring(response.content)
+    except etree.XMLSyntaxError as error:
+        raise PubMedFetchError(f"Malformed PMC XML for {pmcid}: {error}") from error
+
+    article = root.find(".//article")
+    if article is None:
+        raise PubMedFetchError(f"No PMC article returned for {pmcid}")
+    return article
+
+
+def _resolve_ref(client: httpx.Client, ref: PubMedArticleRef) -> PubMedArticleRef:
+    """Fill in a PMCID for a reference that only carries a PMID."""
+    if ref.pmcid or not ref.pmid:
+        return ref
+    resolved = _summarize(client, [ref.pmid])
+    if not resolved:
+        raise PubMedFetchError(f"No PMC full text for PubMed record {ref.pmid}")
+    return resolved[0]
+
+
+def _first_text(node: etree._Element, tag: str) -> str:
+    """Return the normalized text of the first matching descendant."""
+    element = node.find(f".//{tag}")
+    return clean_text("".join(element.itertext()), drop_numeric_citations=False) if element is not None else ""
+
+
+def _article_license(article: etree._Element) -> dict[str, str]:
+    """Extract licensing terms from the article's front matter.
+
+    The PMC Open Access Subset is not uniformly Creative Commons licensed. 88.3%
+    of the guideline slice carries a CC license, found in one of three shapes:
+    an `<ali:license_ref>` element, an `<ext-link>` pointing at
+    creativecommons.org, or a bare URL in the license free text. The name is
+    parsed from the URL path rather than from the `license-type` attribute,
+    which is unusable: the corpus spells it 18 different ways, including
+    `cc-by`, `creativeCommonsBy`, `CC BY-NC-ND`, `open-access` and `openaccess`.
+
+    The remaining 11.7% carry publisher terms that are not equivalent to each
+    other: most are Elsevier's COVID-19 resource centre grant, whose copyright
+    line still reads "All rights reserved"; some are the PMC Open Access
+    Subset's own "unrestricted re-use" statement; and 112 have no `<license>`
+    element at all, only a plain copyright line such as "(c) Springer-Verlag
+    Tokyo 2007".
+    Being in the subset is therefore not by itself a grant to redistribute, so
+    the raw terms are recorded rather than flattened to an empty license.
+    """
+    # The reservation of rights sits in `<copyright-statement>`, a sibling of
+    # `<license>` rather than a child, so reading only the license element loses
+    # the "All rights reserved" half of a COVID-era temporary grant.
+    copyright_statement = _first_text(article, "copyright-statement")
+
+    license_element = article.find(".//license")
+    if license_element is None:
+        return {
+            "license": "",
+            "license_url": "",
+            "license_type": "",
+            "license_statement": "",
+            "copyright_statement": copyright_statement,
+        }
+
+    url = ""
+    license_ref = license_element.find(f".//{_ALI_LICENSE_REF}")
+    if license_ref is not None and license_ref.text:
+        url = license_ref.text.strip()
+    if not url:
+        for element in license_element.iter():
+            href = element.get(_XLINK_HREF) or ""
+            if "creativecommons.org" in href:
+                url = href.strip()
+                break
+    if not url:
+        match = _CC_URL_RE.search("".join(license_element.itertext()))
+        url = match.group(0) if match else ""
+
+    name = ""
+    match = re.search(r"creativecommons\.org/(?:licenses|publicdomain)/([a-z0-9-]+)/", url)
+    if match:
+        token = match.group(1)
+        name = "CC0" if token == "zero" else f"CC {token.upper()}"
+
+    statement = clean_text("".join(license_element.itertext()), drop_numeric_citations=False)
+    return {
+        "license": name,
+        "license_url": url,
+        "license_type": license_element.get("license-type", ""),
+        "license_statement": statement[:_LICENSE_STATEMENT_CHARS],
+        "copyright_statement": copyright_statement,
+    }
+
+
+def _article_figures(article: etree._Element) -> list[dict[str, str]]:
+    """Record each figure's source filename, label and caption.
+
+    The image itself cannot be linked from here. JATS carries only a bare
+    filename (`keag320f1.jpg`), while the served URL inserts a CDN shard and a
+    content hash — `.../pmc/blobs/1ab4/13384732/bd9a2d61ca1a/keag320f1.jpg` —
+    that appear nowhere in the API response. Building a URL from the filename
+    alone yields a 404, which is worse than no link because nothing downstream
+    notices it is broken.
+
+    Other scrapers here keep images as absolute remote references (see the
+    `<img>` handling added in #9 and #10), which works because their sources
+    serve a real path. Recording the filename keeps that door open: a later
+    pass can resolve these against the rendered page or the OA package without
+    re-scraping the corpus.
+
+    Scans the whole article rather than just the body: a graphical abstract is
+    declared in `<front>`, so a body-only scan silently misses it.
+
+    Args:
+        article: The parsed `<article>` element, before tag rewriting.
+    """
+    figures: list[dict[str, str]] = []
+    for figure in article.iter("fig"):
+        graphic = figure.find(".//graphic")
+        href = graphic.get(_XLINK_HREF) if graphic is not None else None
+        if not href:
+            continue
+        figures.append(
+            {
+                "id": figure.get("id", ""),
+                "file": href,
+                "label": _first_text(figure, "label"),
+                "caption": _first_text(figure, "caption")[:_FIGURE_CAPTION_CHARS],
+            }
+        )
+    return figures
+
+
+def _article_supplements(article: etree._Element) -> list[dict[str, str]]:
+    """Record supplementary file names, labels and captions.
+
+    Same situation as `_article_figures`, and measured: across 40 sampled
+    guidelines every supplementary block was a pointer to an external file
+    rather than inline content, totalling 0.18% of body text. The evidence
+    tables a verifier would want live inside those `.docx` and `.tif` payloads,
+    which the API does not return, so the pointer is the only thing worth
+    keeping.
+
+    Args:
+        article: The parsed `<article>` element, before tag rewriting.
+    """
+    supplements: list[dict[str, str]] = []
+    for supplement in article.iter("supplementary-material"):
+        href = supplement.get(_XLINK_HREF)
+        if not href:
+            media = supplement.find(f".//*[@{_XLINK_HREF}]")
+            href = media.get(_XLINK_HREF) if media is not None else None
+        if not href:
+            continue
+        supplements.append(
+            {
+                "id": supplement.get("id", ""),
+                "file": href,
+                "label": _first_text(supplement, "label"),
+                "caption": _first_text(supplement, "caption")[:_FIGURE_CAPTION_CHARS],
+            }
+        )
+    return supplements
+
+
+def _heading_depth(section: etree._Element) -> int:
+    """Return how deeply a `<sec>` is nested inside the article body."""
+    depth = 1
+    parent = section.getparent()
+    while parent is not None:
+        if parent.tag == "sec":
+            depth += 1
+        parent = parent.getparent()
+    return depth
+
+
+def _jats_to_html(source: etree._Element) -> str:
+    """Rewrite a JATS subtree into HTML that `html_to_markdown` understands.
+
+    JATS is close enough to HTML that renaming tags is cheaper and less
+    error-prone than writing a second markdown serializer: `<table-wrap>`
+    already contains genuine XHTML tables, and inline markup maps one-to-one.
+    """
+    root = copy.deepcopy(source)
+    etree.strip_elements(root, *_DROPPED_TAGS, with_tail=False)
+    etree.strip_tags(root, *_UNWRAPPED_TAGS)
+
+    # An `<abstract>` labels itself "Abstract", which would duplicate the heading
+    # this module already writes above it.
+    if root.tag == "abstract":
+        title = root.find("title")
+        if title is not None:
+            root.remove(title)
+
+    # Headings depend on `<sec>` nesting, so resolve them before `sec` is renamed.
+    for section in root.iter("sec"):
+        title = section.find("title")
+        if title is not None:
+            title.tag = f"h{min(_heading_depth(section) + 1, 6)}"
+    for title in root.iter("title"):
+        title.tag = "strong"
+
+    for element in root.iter():
+        if not isinstance(element.tag, str):
+            continue
+        if element.tag == "list":
+            element.tag = "ol" if element.get("list-type") == "order" else "ul"
+        elif element.tag == "ext-link":
+            href = element.get(_XLINK_HREF)
+            element.tag = "a"
+            if href:
+                element.set("href", href)
+        else:
+            element.tag = _RENAMED_TAGS.get(element.tag, element.tag)
+        for name in list(element.attrib):
+            if name.startswith("{") or name in {"list-type", "content-type", "position"}:
+                del element.attrib[name]
+
+    return etree.tostring(root, encoding="unicode", method="html")
+
+
+def _jats_to_markdown(source: etree._Element, *, link_mode: LinkMode) -> str:
+    """Convert a JATS subtree to markdown and close up gaps left by dropped nodes.
+
+    The spacing pass has to run last: `html_to_markdown` is what removes the
+    "[12]" text of an unwrapped `xref`, so the orphaned space only appears once
+    the markdown exists.
+    """
+    markdown = html_to_markdown(_jats_to_html(source), link_mode=link_mode)
+    return _ORPHAN_SPACE_RE.sub(r"\1", markdown)
+
+
+def build_pubmed_article_text(
+    client: httpx.Client,
+    ref: PubMedArticleRef,
+    *,
+    link_mode: LinkMode = LinkMode.KEEP,
+) -> tuple[str, int, str, dict[str, Any]]:
+    """Scrape one guideline's full text into markdown, section count, title and metadata.
+
+    Args:
+        client: HTTP client used to call E-utilities.
+        ref: PubMed guideline reference to scrape.
+        link_mode: Whether links are kept as markdown links or stripped to their
+            visible text (default: LinkMode.KEEP).
+    """
+    ref = _resolve_ref(client, ref)
+    article = _fetch_pmc_article(client, ref.pmcid)
+
+    body = article.find(".//body")
+    if body is None:
+        # Roughly 2% of the subset is deposited as metadata only. That is a
+        # property of the record, not a fetch failure.
+        raise PubMedFetchError(f"No full text body in PMC record {ref.pmcid}")
+
+    sections: list[str] = []
+    abstract = article.find(".//abstract")
+    if abstract is not None:
+        abstract_markdown = _jats_to_markdown(abstract, link_mode=link_mode)
+        if abstract_markdown:
+            sections.append(f"## Abstract\n\n{abstract_markdown}")
+
+    body_markdown = _jats_to_markdown(body, link_mode=link_mode)
+    if body_markdown:
+        sections.append(body_markdown)
+
+    content = "\n\n".join(sections).strip()
+    if not content:
+        raise PubMedFetchError(f"No readable content in PMC record {ref.pmcid}")
+
+    title = ref.title or _first_text(article, "article-title") or ref.slug
+    section_count = max(len(body.findall("sec")), 1) + (1 if abstract is not None else 0)
+    metadata = {
+        "pmid": ref.pmid or _pmid_from_article(article),
+        "pmcid": ref.pmcid,
+        "doi": ref.doi or _article_id(article, "doi"),
+        "journal": ref.journal or _first_text(article, "journal-title"),
+        "pub_date": ref.pub_date,
+        "article_type": article.get("article-type", ""),
+        **_article_license(article),
+        "figures": _article_figures(article),
+        "supplements": _article_supplements(article),
+        "content_length_chars": len(content),
+    }
+    return content, section_count, title, metadata
+
+
+def _article_id(article: etree._Element, id_type: str) -> str:
+    """Return an `<article-id>` value of the requested type."""
+    for element in article.iter("article-id"):
+        if element.get("pub-id-type") == id_type:
+            return "".join(element.itertext()).strip()
+    return ""
+
+
+def _pmid_from_article(article: etree._Element) -> str:
+    return _article_id(article, "pmid")
+
+
+def scrape_pubmed_article(
+    client: httpx.Client,
+    ref: PubMedArticleRef,
+    *,
+    link_mode: LinkMode = LinkMode.KEEP,
+) -> ScrapedDocument:
+    """Scrape a PubMed guideline into a normalized document.
+
+    Args:
+        client: HTTP client used to call E-utilities.
+        ref: PubMed guideline reference to scrape.
+        link_mode: Whether links are kept as markdown links or stripped to their
+            visible text (default: LinkMode.KEEP).
+    """
+    content, section_count, title, metadata = build_pubmed_article_text(client, ref, link_mode=link_mode)
+    # Prefer the PMID from the record itself so one article gets one identifier
+    # whether it was reached from the listing or from a PMC URL, which carries
+    # no PMID of its own.
+    slug = metadata["pmid"] or ref.slug or metadata["pmcid"]
+    return ScrapedDocument(
+        source="pubmed",
+        external_id=f"pubmed-{slug}",
+        title=title,
+        url=ref.page_url,
+        content=content,
+        section_count=section_count,
+        metadata=metadata,
+    )
+
+
+def _scrape_or_skip(
+    client: httpx.Client,
+    ref: PubMedArticleRef,
+    *,
+    link_mode: LinkMode,
+) -> ScrapedDocument | None:
+    """Scrape one guideline, skipping records PMC holds without full text.
+
+    About 2% of the open-access subset is deposited as metadata only, so a run
+    over the whole corpus should not die when it reaches one.
+
+    Args:
+        client: HTTP client used to call E-utilities.
+        ref: PubMed guideline reference to scrape.
+        link_mode: Whether links are kept as markdown links or stripped to their
+            visible text.
+    """
+    try:
+        return scrape_pubmed_article(client, ref, link_mode=link_mode)
+    except PubMedFetchError:
+        logger.warning("Skipping PubMed guideline %r", ref.slug, exc_info=True)
+        return None
+
+
+def scrape_pubmed(
+    *,
+    documents: int | None,
+    link_mode: LinkMode = LinkMode.KEEP,
+    url: str | None = None,
+) -> ScrapeRun:
+    """Scrape PubMed guidelines from a URL or the open-access search listing.
+
+    Args:
+        documents: Number of documents to scrape. Ignored when `url` is set.
+            When unset, listing pages are fetched until a page returns no items
+            (default: None).
+        link_mode: Whether links are kept as markdown links or stripped to their
+            visible text (default: LinkMode.KEEP).
+        url: PubMed or PMC article URL to scrape as a single document
+            (default: None).
+    """
+    if url is not None:
+
+        def scrape_url() -> Iterable[ScrapedDocument]:
+            with default_client() as client:
+                yield scrape_pubmed_article(client, pubmed_ref_from_url(url), link_mode=link_mode)
+
+        return ScrapeRun(documents=scrape_url(), total=1)
+
+    with default_client() as client:
+        first_page = list_pubmed_guidelines(client, page=1)
+    total = first_page.total if documents is None or first_page.total is None else min(documents, first_page.total)
+    return ScrapeRun(
+        total=total,
+        documents=scrape_listing_documents(
+            documents=documents,
+            client_factory=default_client,
+            first_page_items=first_page.refs,
+            list_page=lambda client, page: list_pubmed_guidelines(client, page).refs,
+            scrape_item=lambda client, ref: _scrape_or_skip(client, ref, link_mode=link_mode),
+            document_delay_seconds=DOCUMENT_DELAY_SECONDS,
+        ),
+    )
+
+
+__all__ = [
+    "BASE_URL",
+    "DOCUMENT_DELAY_SECONDS",
+    "EUTILS_URL",
+    "PMC_ARTICLE_URL",
+    "PUBMED_DATASET_DISPLAY_NAME",
+    "PUBMED_DATASET_NAME",
+    "SEARCH_TERM",
+    "PubMedArticleRef",
+    "PubMedFetchError",
+    "PubMedListingPage",
+    "build_pubmed_article_text",
+    "list_pubmed_guidelines",
+    "pubmed_ref_from_url",
+    "scrape_pubmed",
+    "scrape_pubmed_article",
+]
diff --git a/datasets/test/test_scraping_pubmed.py b/datasets/test/test_scraping_pubmed.py
new file mode 100644
index 00000000..16deaaba
--- /dev/null
+++ b/datasets/test/test_scraping_pubmed.py
@@ -0,0 +1,478 @@
+"""Tests for PubMed scraping helpers."""
+
+import json
+
+import httpx
+import pytest
+
+from amfv_datasets.scraping import base
+from amfv_datasets.scraping.html import LinkMode
+from amfv_datasets.scraping.pubmed import (
+    LISTING_PAGE_SIZE,
+    PubMedArticleRef,
+    PubMedFetchError,
+    build_pubmed_article_text,
+    list_pubmed_guidelines,
+    pubmed_ref_from_url,
+    scrape_pubmed,
+    scrape_pubmed_article,
+)
+
+_BODY = """
+<body>
+  <sec><title>Scope and purpose</title>
+    <p>This guideline covers <italic>adult</italic> hypertension.</p>
+    <sec><title>Guideline objectives</title><p>Reduce cardiovascular risk.</p></sec>
+  </sec>
+  <sec><title>Recommendations</title>
+    <p>Treat to target <xref ref-type="bibr" rid="r1">[1]</xref> .</p>
+    <table-wrap><label>Table 1</label>
+      <caption><title>Thresholds</title></caption>
+      <table>
+        <thead><tr><th>Stage</th><th>Threshold</th></tr></thead>
+        <tbody><tr><td>Stage 2</td><td>140/90 mmHg</td></tr></tbody>
+      </table>
+    </table-wrap>
+    <list list-type="order"><list-item><p>Measure blood pressure.</p></list-item></list>
+  </sec>
+  <sec><title>Treatment algorithm</title>
+    <fig id="g1-F1"><label>Figure 1</label>
+      <caption><p>Stepped care algorithm for resistant hypertension.</p></caption>
+      <graphic xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="g1f1.jpg"/>
+    </fig>
+  </sec>
+  <supplementary-material content-type="local-data" id="MOESM1">
+    <media xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="12345_MOESM1_ESM.docx"/>
+    <caption><p>Additional file 1: Table S1. Evidence profile for each recommendation.</p></caption>
+  </supplementary-material>
+  <ref-list><ref id="r1"><mixed-citation>Smith J. Irrelevant citation.</mixed-citation></ref></ref-list>
+</body>
+"""
+
+_LICENSE_ALI = (
+    "<license><ali:license_ref>https://creativecommons.org/licenses/by/4.0/</ali:license_ref>"
+    "<license-p>Open access.</license-p></license>"
+)
+_LICENSE_EXT_LINK = (
+    "<license><license-p>Distributed under the "
+    '<ext-link ext-link-type="uri" xlink:href="https://creativecommons.org/licenses/by-nc/4.0/">'
+    "CC BY-NC</ext-link> license.</license-p></license>"
+)
+_LICENSE_TEXT_ONLY = (
+    "<license><license-p>Distributed under "
+    "https://creativecommons.org/licenses/by-nc-nd/4.0/ terms.</license-p></license>"
+)
+# 11.7% of the open-access subset carries publisher terms instead of a CC
+# license, and they are not equivalent to each other.
+_LICENSE_NON_CC_PERMISSIVE = (
+    '<license license-type="open-access"><license-p>This article is made available via the '
+    "PMC Open Access Subset for unrestricted re-use and analyses in any form or by any means "
+    "with acknowledgement of the original source.</license-p></license>"
+)
+_LICENSE_NON_CC_RESERVED = (
+    "<copyright-statement>© 2020 Elsevier Masson SAS. All rights reserved.</copyright-statement>"
+    "<license><license-p>Since January 2020 Elsevier has created a COVID-19 resource centre "
+    "with free information.</license-p></license>"
+)
+
+
+def _jats(*, body: str = _BODY, license_block: str = _LICENSE_ALI, abstract: str = "") -> str:
+    """Build a minimal but structurally faithful PMC JATS response."""
+    return (
+        '<?xml version="1.0"?>'
+        '<pmc-articleset><article article-type="review-article"'
+        ' xmlns:xlink="http://www.w3.org/1999/xlink"'
+        ' xmlns:ali="http://www.niso.org/schemas/ali/1.0/">'
+        "<front>"
+        "<journal-meta><journal-title>Journal of Guidelines</journal-title></journal-meta>"
+        "<article-meta>"
+        '<article-id pub-id-type="pmid">42476581</article-id>'
+        '<article-id pub-id-type="pmcid">PMC13384732</article-id>'
+        '<article-id pub-id-type="doi">10.1093/example.1</article-id>'
+        "<title-group><article-title>Hypertension guideline</article-title></title-group>"
+        f"<permissions>{license_block}</permissions>"
+        f"{abstract}"
+        "</article-meta></front>"
+        f"{body}"
+        "</article></pmc-articleset>"
+    )
+
+
+def _article_client(xml: str) -> httpx.Client:
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert request.url.path.endswith("efetch.fcgi")
+        assert request.url.params["db"] == "pmc"
+        # NCBI's usage policy asks every request to name the calling tool.
+        assert request.url.params["tool"] == "amfv"
+        return httpx.Response(200, content=xml.encode())
+
+    return httpx.Client(transport=httpx.MockTransport(handler))
+
+
+def _listing_handler(
+    pages: dict[int, list[str]],
+    *,
+    total: int,
+    summaries: dict[str, dict] | None = None,
+    seen_offsets: list[int] | None = None,
+):
+    """Serve esearch/esummary/efetch for a fake corpus keyed by PMID."""
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        path = request.url.path
+        params = request.url.params
+        if path.endswith("esearch.fcgi"):
+            offset = int(params["retstart"])
+            if seen_offsets is not None:
+                seen_offsets.append(offset)
+            page = offset // LISTING_PAGE_SIZE + 1
+            return httpx.Response(
+                200,
+                text=json.dumps({"esearchresult": {"count": str(total), "idlist": pages.get(page, [])}}),
+            )
+        if path.endswith("esummary.fcgi"):
+            uids = params["id"].split(",")
+            result = {"uids": uids}
+            for uid in uids:
+                result[uid] = (summaries or {}).get(
+                    uid,
+                    {
+                        "title": f"Guideline {uid}",
+                        "fulljournalname": "Journal of Guidelines",
+                        "pubdate": "2026 Jan 1",
+                        "articleids": [
+                            {"idtype": "pubmed", "value": uid},
+                            {"idtype": "doi", "value": f"10.1000/{uid}"},
+                            {"idtype": "pmc", "value": f"PMC{uid}"},
+                        ],
+                    },
+                )
+            return httpx.Response(200, text=json.dumps({"result": result}))
+        pmcid = params["id"]
+        if pmcid == "999":
+            # PMC holds the record but no full text was deposited.
+            return httpx.Response(200, content=_jats(body="").encode())
+        return httpx.Response(200, content=_jats().encode())
+
+    return handler
+
+
+def test_list_pubmed_guidelines_resolves_pmids_to_refs_with_pmcids() -> None:
+    """One esearch plus one esummary yields refs carrying full citation metadata."""
+    client = httpx.Client(transport=httpx.MockTransport(_listing_handler({1: ["111", "222"]}, total=2)))
+
+    listing = list_pubmed_guidelines(client)
+
+    assert listing.total == 2
+    assert [ref.pmid for ref in listing.refs] == ["111", "222"]
+    assert listing.refs[0].pmcid == "PMC111"
+    assert listing.refs[0].title == "Guideline 111"
+    assert listing.refs[0].doi == "10.1000/111"
+    assert listing.refs[0].page_url == "https://pubmed.ncbi.nlm.nih.gov/111/"
+
+
+def test_list_pubmed_guidelines_pages_by_numeric_offset() -> None:
+    """Page number maps onto `retstart`, which is what makes paging work at all."""
+    seen: list[int] = []
+    client = httpx.Client(transport=httpx.MockTransport(_listing_handler({3: ["333"]}, total=900, seen_offsets=seen)))
+
+    listing = list_pubmed_guidelines(client, page=3)
+
+    assert seen == [2 * LISTING_PAGE_SIZE]
+    assert [ref.pmid for ref in listing.refs] == ["333"]
+
+
+def test_list_pubmed_guidelines_returns_no_refs_past_the_end_of_the_listing() -> None:
+    """An empty page terminates the scrape loop rather than raising."""
+    client = httpx.Client(transport=httpx.MockTransport(_listing_handler({}, total=2)))
+
+    listing = list_pubmed_guidelines(client, page=99)
+
+    assert listing.refs == []
+    assert listing.total == 2
+
+
+def test_list_pubmed_guidelines_skips_records_with_no_pmc_copy() -> None:
+    """The open-access filter should guarantee a PMC id; a record without one is dropped."""
+    summaries = {
+        "111": {"title": "No full text", "articleids": [{"idtype": "pubmed", "value": "111"}]},
+    }
+    client = httpx.Client(
+        transport=httpx.MockTransport(_listing_handler({1: ["111", "222"]}, total=2, summaries=summaries))
+    )
+
+    listing = list_pubmed_guidelines(client)
+
+    assert [ref.pmid for ref in listing.refs] == ["222"]
+
+
+def test_build_pubmed_article_text_maps_jats_sections_to_nested_headings() -> None:
+    """`<sec>` nesting depth becomes markdown heading depth."""
+    content, section_count, title, metadata = build_pubmed_article_text(
+        _article_client(_jats()),
+        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
+        link_mode=LinkMode.STRIP,
+    )
+
+    assert "## Scope and purpose" in content
+    assert "### Guideline objectives" in content
+    assert "## Recommendations" in content
+    assert title == "Hypertension guideline"
+    assert section_count == 3
+    assert metadata["journal"] == "Journal of Guidelines"
+    assert metadata["article_type"] == "review-article"
+    assert metadata["content_length_chars"] == len(content)
+
+
+def test_build_pubmed_article_text_keeps_clinical_tables_and_drops_citation_pointers() -> None:
+    """Recommendation thresholds must survive; `xref` markers and reference lists must not."""
+    content, _, _, _ = build_pubmed_article_text(
+        _article_client(_jats()),
+        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
+    )
+
+    assert "140/90 mmHg" in content
+    assert "Measure blood pressure" in content
+    # The `[1]` pointer and the reference list it points at are citation
+    # apparatus, not clinical content.
+    assert "[1]" not in content
+    assert "Irrelevant citation" not in content
+    # Unwrapping the pointer must not leave " ." behind.
+    assert "Treat to target ." not in content
+    assert "Treat to target." in content
+
+
+def test_build_pubmed_article_text_records_figures_it_cannot_link() -> None:
+    """The image URL is not derivable from the API, so keep the filename for a later pass."""
+    content, _, _, metadata = build_pubmed_article_text(
+        _article_client(_jats()),
+        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
+    )
+
+    assert metadata["figures"] == [
+        {
+            "id": "g1-F1",
+            "file": "g1f1.jpg",
+            "label": "Figure 1",
+            "caption": "Stepped care algorithm for resistant hypertension.",
+        }
+    ]
+    # The caption stays readable in the text; only the unlinkable image goes.
+    assert "Stepped care algorithm" in content
+    assert "g1f1.jpg" not in content
+
+
+def test_build_pubmed_article_text_records_supplementary_files() -> None:
+    """Evidence tables live in supplementary .docx payloads the API does not return."""
+    content, _, _, metadata = build_pubmed_article_text(
+        _article_client(_jats()),
+        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
+    )
+
+    assert metadata["supplements"] == [
+        {
+            "id": "MOESM1",
+            "file": "12345_MOESM1_ESM.docx",
+            "label": "",
+            "caption": "Additional file 1: Table S1. Evidence profile for each recommendation.",
+        }
+    ]
+    assert "12345_MOESM1_ESM.docx" not in content
+
+
+def test_build_pubmed_article_text_does_not_repeat_the_abstract_heading() -> None:
+    """JATS labels the abstract itself, which would duplicate the heading we add."""
+    content, section_count, _, _ = build_pubmed_article_text(
+        _article_client(_jats(abstract="<abstract><title>Abstract</title><p>Summary text.</p></abstract>")),
+        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
+    )
+
+    assert content.startswith("## Abstract")
+    assert "**Abstract**" not in content
+    assert "Summary text." in content
+    # Three body sections plus the abstract.
+    assert section_count == 4
+
+
+@pytest.mark.parametrize(
+    ("license_block", "expected_name", "expected_url"),
+    [
+        (_LICENSE_ALI, "CC BY", "https://creativecommons.org/licenses/by/4.0/"),
+        (_LICENSE_EXT_LINK, "CC BY-NC", "https://creativecommons.org/licenses/by-nc/4.0/"),
+        (_LICENSE_TEXT_ONLY, "CC BY-NC-ND", "https://creativecommons.org/licenses/by-nc-nd/4.0/"),
+    ],
+)
+def test_build_pubmed_article_text_reads_every_license_shape(
+    license_block: str, expected_name: str, expected_url: str
+) -> None:
+    """All three shapes occur in the open-access subset, and ND changes what we may redistribute."""
+    _, _, _, metadata = build_pubmed_article_text(
+        _article_client(_jats(license_block=license_block)),
+        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
+    )
+
+    assert metadata["license"] == expected_name
+    assert metadata["license_url"] == expected_url
+
+
+def test_build_pubmed_article_text_records_permissive_publisher_terms() -> None:
+    """A publisher license is not a missing license, so the terms are kept verbatim."""
+    _, _, _, metadata = build_pubmed_article_text(
+        _article_client(_jats(license_block=_LICENSE_NON_CC_PERMISSIVE)),
+        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
+    )
+
+    assert metadata["license"] == ""
+    assert metadata["license_url"] == ""
+    assert metadata["license_type"] == "open-access"
+    assert "unrestricted re-use" in metadata["license_statement"]
+
+
+def test_build_pubmed_article_text_records_a_reserved_copyright_statement() -> None:
+    """COVID-era free access still reserves all rights, and that text is not inside `<license>`."""
+    _, _, _, metadata = build_pubmed_article_text(
+        _article_client(_jats(license_block=_LICENSE_NON_CC_RESERVED)),
+        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
+    )
+
+    assert metadata["license"] == ""
+    # Reading only `<license>` would report free access and lose the reservation.
+    assert "free information" in metadata["license_statement"]
+    assert "All rights reserved" in metadata["copyright_statement"]
+
+
+def test_build_pubmed_article_text_rejects_a_metadata_only_record() -> None:
+    """About 2% of the subset has no deposited full text."""
+    with pytest.raises(PubMedFetchError):
+        build_pubmed_article_text(
+            _article_client(_jats(body="")),
+            PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
+        )
+
+
+def test_scrape_pubmed_article_normalizes_into_a_scraped_document() -> None:
+    """Scraped guidelines use the shared source/external_id conventions."""
+    document = scrape_pubmed_article(
+        _article_client(_jats()),
+        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
+    )
+
+    assert document.source == "pubmed"
+    assert document.external_id == "pubmed-42476581"
+    assert document.url == "https://pubmed.ncbi.nlm.nih.gov/42476581/"
+    assert document.metadata["pmcid"] == "PMC13384732"
+    assert "Scope and purpose" in document.content
+
+
+def test_scrape_pubmed_pages_through_the_listing_and_terminates(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Paging advances by offset, crosses a page boundary, and stops on the empty page."""
+    seen: list[int] = []
+    handler = _listing_handler({1: ["111"], 2: ["222"]}, total=2, seen_offsets=seen)
+    monkeypatch.setattr(
+        "amfv_datasets.scraping.pubmed.default_client",
+        lambda: httpx.Client(transport=httpx.MockTransport(handler)),
+    )
+    monkeypatch.setattr(base.time, "sleep", lambda _seconds: None)
+
+    run = scrape_pubmed(documents=None)
+    documents = list(run.documents)
+
+    assert run.total == 2
+    assert [document.external_id for document in documents] == ["pubmed-111", "pubmed-222"]
+    # Page 1 is fetched once for the total and reused via `first_page_items`,
+    # so the listing is not re-requested at offset 0 during the run.
+    assert seen == [0, LISTING_PAGE_SIZE, 2 * LISTING_PAGE_SIZE]
+
+
+def test_scrape_pubmed_skips_records_pmc_holds_without_full_text(monkeypatch: pytest.MonkeyPatch) -> None:
+    """One metadata-only record does not abort a run over the rest of the listing."""
+    handler = _listing_handler({1: ["999", "222"]}, total=2)
+    monkeypatch.setattr(
+        "amfv_datasets.scraping.pubmed.default_client",
+        lambda: httpx.Client(transport=httpx.MockTransport(handler)),
+    )
+    monkeypatch.setattr(base.time, "sleep", lambda _seconds: None)
+
+    documents = list(scrape_pubmed(documents=None).documents)
+
+    assert [document.external_id for document in documents] == ["pubmed-222"]
+
+
+def test_eutils_retries_when_ncbi_rate_limits(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A 429 mid-run must not kill the scrape; NCBI returns them on bursts."""
+    monkeypatch.setattr("amfv_datasets.scraping.pubmed.time.sleep", lambda _seconds: None)
+    attempts = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        attempts.append(request.url.path)
+        if len(attempts) < 3:
+            return httpx.Response(429, text="Too Many Requests")
+        return httpx.Response(200, content=_jats().encode())
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    document = scrape_pubmed_article(client, PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"))
+
+    assert len(attempts) == 3
+    assert document.external_id == "pubmed-42476581"
+
+
+def test_eutils_gives_up_after_repeated_rate_limits(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Persistent 429s are a real stop signal, not something to retry forever."""
+    monkeypatch.setattr("amfv_datasets.scraping.pubmed.time.sleep", lambda _seconds: None)
+    attempts = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        attempts.append(request.url.path)
+        return httpx.Response(429, text="Too Many Requests")
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    with pytest.raises(httpx.HTTPStatusError):
+        scrape_pubmed_article(client, PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"))
+
+    assert len(attempts) == 4
+
+
+def test_pubmed_ref_from_url_accepts_both_pubmed_and_pmc_urls() -> None:
+    """Only one identifier is knowable from a URL; the other is resolved when scraping."""
+    pubmed_ref = pubmed_ref_from_url("https://pubmed.ncbi.nlm.nih.gov/42476581/")
+    pmc_ref = pubmed_ref_from_url("https://www.ncbi.nlm.nih.gov/pmc/articles/PMC13384732/")
+
+    assert pubmed_ref == PubMedArticleRef(pmid="42476581")
+    assert pmc_ref == PubMedArticleRef(pmcid="PMC13384732")
+    assert pmc_ref.page_url == "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC13384732/"
+
+
+def test_scrape_pubmed_article_uses_the_same_id_from_either_entry_point() -> None:
+    """A PMC URL carries no PMID, but the record does, so both routes agree."""
+    from_listing = scrape_pubmed_article(
+        _article_client(_jats()), PubMedArticleRef(pmid="42476581", pmcid="PMC13384732")
+    )
+    from_pmc_url = scrape_pubmed_article(_article_client(_jats()), PubMedArticleRef(pmcid="PMC13384732"))
+
+    assert from_listing.external_id == from_pmc_url.external_id == "pubmed-42476581"
+
+
+def test_pubmed_ref_from_url_resolves_a_pmid_to_full_text(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A PubMed URL carries no PMCID, so efetch needs one looked up first."""
+    handler = _listing_handler({1: ["111"]}, total=1)
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+
+    document = scrape_pubmed_article(client, pubmed_ref_from_url("https://pubmed.ncbi.nlm.nih.gov/111/"))
+
+    assert document.external_id == "pubmed-111"
+    assert document.metadata["pmcid"] == "PMC111"
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "https://example.org/42476581/",
+        "https://pubmed.ncbi.nlm.nih.gov/not-a-pmid/",
+        "https://www.ncbi.nlm.nih.gov/pmc/articles/13384732/",
+        "ftp://pubmed.ncbi.nlm.nih.gov/42476581/",
+    ],
+)
+def test_pubmed_ref_from_url_rejects_non_article_urls(url: str) -> None:
+    """Anything that is not a PubMed or PMC article URL is an error, not a guess."""
+    with pytest.raises(PubMedFetchError):
+        pubmed_ref_from_url(url)

From f83314d4159c9c15c23ac9197a639624207d4153 Mon Sep 17 00:00:00 2001
From: Zander Giuffrida <zndr27@gmail.com>
Date: Sun, 26 Jul 2026 00:05:17 -0700
Subject: [PATCH 08/15] allow scrape_item to skip a listed item

Taken verbatim from #10 so the two merge cleanly in either order. A listing
entry is not always a document: PMC holds some open-access records as metadata
only, with no full text deposited. Returning None skips the entry without
consuming the --documents budget, and the delay gate moves from scraped to
attempted so skipped items still rate-limit.

Drop this commit if #10 lands first.

(cherry picked from commit b55d196af48bf57905560acc792c774102a6dbff)
---
 datasets/amfv_datasets/scraping/base.py | 14 ++++++---
 datasets/test/test_scraping_base.py     | 40 +++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/datasets/amfv_datasets/scraping/base.py b/datasets/amfv_datasets/scraping/base.py
index 0708ee87..3651cdb7 100644
--- a/datasets/amfv_datasets/scraping/base.py
+++ b/datasets/amfv_datasets/scraping/base.py
@@ -74,7 +74,7 @@ def scrape_listing_documents[ListingItemT](
     documents: int | None,
     client_factory: Callable[[], AbstractContextManager[httpx.Client]],
     list_page: Callable[[httpx.Client, int], Iterable[ListingItemT]],
-    scrape_item: Callable[[httpx.Client, ListingItemT], ScrapedDocument],
+    scrape_item: Callable[[httpx.Client, ListingItemT], ScrapedDocument | None],
     document_delay_seconds: float = 5.0,
     first_page_items: Iterable[ListingItemT] | None = None,
 ) -> Iterable[ScrapedDocument]:
@@ -85,7 +85,8 @@ def scrape_listing_documents[ListingItemT](
             fetched until a page returns no items (default: None).
         client_factory: Factory returning a context-managed HTTP client.
         list_page: Function that lists source-specific items for a page.
-        scrape_item: Function that scrapes one listed item into a document.
+        scrape_item: Function that scrapes one listed item into a document. It
+            may return None to skip a discovered item that is not a document.
         document_delay_seconds: Delay before scraping each document after the
             first one (default: 5.0).
         first_page_items: Already-fetched first listing page items. When set,
@@ -99,6 +100,7 @@ def scrape_listing_documents[ListingItemT](
     with client_factory() as client:
         page = 1
         scraped = 0
+        attempted = 0
         page_items = list(first_page_items) if first_page_items is not None else None
         while documents is None or scraped < documents:
             if page_items is None:
@@ -111,9 +113,13 @@ def scrape_listing_documents[ListingItemT](
             for item in items:
                 if documents is not None and scraped >= documents:
                     break
-                if scraped and document_delay_seconds:
+                if attempted and document_delay_seconds:
                     time.sleep(document_delay_seconds)
-                yield scrape_item(client, item)
+                document = scrape_item(client, item)
+                attempted += 1
+                if document is None:
+                    continue
+                yield document
                 scraped += 1
             page += 1
 
diff --git a/datasets/test/test_scraping_base.py b/datasets/test/test_scraping_base.py
index eed3c838..2201e4de 100644
--- a/datasets/test/test_scraping_base.py
+++ b/datasets/test/test_scraping_base.py
@@ -149,3 +149,43 @@ def scrape_item(client: httpx.Client, item: str) -> ScrapedDocument:
 
     assert [document.external_id for document in documents] == ["item-1", "item-2", "item-3"]
     assert delays == [5.0, 5.0]
+
+
+def test_scrape_listing_documents_skips_non_documents_without_consuming_limit(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Skipped listing entries are delayed but do not count as returned documents."""
+    delays: list[float] = []
+
+    class _FakeClient:
+        def __enter__(self) -> "_FakeClient":
+            return self
+
+        def __exit__(self, *args: object) -> None:
+            return None
+
+    def client_factory():
+        return _FakeClient()
+
+    def list_page(client: httpx.Client, page: int) -> list[str]:
+        return ["skip", "item-1", "item-2"] if page == 1 else []
+
+    def scrape_item(client: httpx.Client, item: str) -> ScrapedDocument | None:
+        if item == "skip":
+            return None
+        return ScrapedDocument("test", item, item, f"https://example.org/{item}", "content")
+
+    monkeypatch.setattr(base.time, "sleep", delays.append)
+
+    documents = list(
+        scrape_listing_documents(
+            documents=2,
+            client_factory=client_factory,
+            list_page=list_page,
+            scrape_item=scrape_item,
+            document_delay_seconds=5.0,
+        )
+    )
+
+    assert [document.external_id for document in documents] == ["item-1", "item-2"]
+    assert delays == [5.0, 5.0]

From 7a806463d90b462e60e5034153c3d237dd3cf5d9 Mon Sep 17 00:00:00 2001
From: ahoblitz <ahoblitz@gmail.com>
Date: Sat, 8 Aug 2026 09:38:56 -0700
Subject: [PATCH 09/15] absolutize img src in html_to_markdown

Taken as written from the CPS and RCH scraper PRs by mkrastev (#9, #10), where this change appears identically inside larger commits and could not be cherry-picked on its own.
---
 datasets/amfv_datasets/scraping/html.py | 2 ++
 datasets/test/test_scraping_html.py     | 9 +++++++++
 2 files changed, 11 insertions(+)

diff --git a/datasets/amfv_datasets/scraping/html.py b/datasets/amfv_datasets/scraping/html.py
index 2bd1c968..15805281 100644
--- a/datasets/amfv_datasets/scraping/html.py
+++ b/datasets/amfv_datasets/scraping/html.py
@@ -119,6 +119,8 @@ def _absolutize_links(html_text: str, *, base_url: str) -> str:
     root = lxml_html.fragment_fromstring(html_text, create_parent="div")
     for link in root.xpath(".//a[@href]"):
         link.set("href", urljoin(base_url, link.get("href")))
+    for image in root.xpath(".//img[@src]"):
+        image.set("src", urljoin(base_url, image.get("src")))
     return "".join(lxml_html.tostring(child, encoding="unicode") for child in root)
 
 
diff --git a/datasets/test/test_scraping_html.py b/datasets/test/test_scraping_html.py
index d34a8fa3..b228222d 100644
--- a/datasets/test/test_scraping_html.py
+++ b/datasets/test/test_scraping_html.py
@@ -68,3 +68,12 @@ def test_html_to_markdown_can_strip_links() -> None:
     html_text = '<p>Offer <a href="https://example.org">treatment</a>.</p>'
 
     assert html_to_markdown(html_text, link_mode=LinkMode.STRIP) == "Offer treatment."
+
+
+def test_html_to_markdown_absolutizes_images() -> None:
+    """Relative image sources are preserved as absolute markdown image URLs."""
+    html_text = '<img src="/images/flowchart.png" alt="Treatment flowchart">'
+
+    assert html_to_markdown(html_text, base_url="https://example.org/guideline/") == (
+        "![Treatment flowchart](https://example.org/images/flowchart.png)"
+    )

From fdd66eb2320d18e308c89ee124dd92ab1031aed8 Mon Sep 17 00:00:00 2001
From: ahoblitz <ahoblitz@gmail.com>
Date: Sat, 8 Aug 2026 10:41:49 -0700
Subject: [PATCH 10/15] register the merged scrapers in the CLI

---
 datasets/amfv_datasets/scraping/__init__.py | 63 ++++++++++++++++++---
 datasets/amfv_datasets/scraping/cli.py      | 18 +++++-
 2 files changed, 71 insertions(+), 10 deletions(-)

diff --git a/datasets/amfv_datasets/scraping/__init__.py b/datasets/amfv_datasets/scraping/__init__.py
index 5cadb8e4..f48815b6 100644
--- a/datasets/amfv_datasets/scraping/__init__.py
+++ b/datasets/amfv_datasets/scraping/__init__.py
@@ -1,14 +1,27 @@
 """Web scraping helpers and source-specific scrapers."""
 
 from amfv_datasets.scraping.base import (
-    USER_AGENT,
-    ScrapedDocument,
     ScrapeError,
     ScrapeRun,
+    ScrapedDocument,
+    USER_AGENT,
     default_client,
     scrape_listing_documents,
 )
-from amfv_datasets.scraping.cli import OutputFormat, ScraperSource
+from amfv_datasets.scraping.cli import (
+    OutputFormat,
+    ScraperSource,
+)
+from amfv_datasets.scraping.cps import (
+    CpsFetchError,
+    CpsStatementRef,
+    build_statement_text,
+    list_statements,
+    listing_page_url,
+    scrape_cps,
+    scrape_statement,
+    statement_ref_from_url,
+)
 from amfv_datasets.scraping.html import (
     LinkMode,
     absolute_unique_urls,
@@ -22,10 +35,8 @@
     IDSAGuidelineRef,
     idsa_ref_from_url,
     list_practice_guidelines,
-    scrape_idsa,
-)
-from amfv_datasets.scraping.idsa import (
     scrape_guideline as scrape_idsa_guideline,
+    scrape_idsa,
 )
 from amfv_datasets.scraping.nice import (
     GuidanceListingPage,
@@ -37,6 +48,21 @@
     scrape_guideline,
     scrape_nice,
 )
+from amfv_datasets.scraping.pubmed import (
+    PubMedArticleRef,
+    PubMedFetchError,
+    PubMedListingPage,
+    build_pubmed_article_text,
+    list_pubmed_guidelines,
+    pubmed_ref_from_url,
+    scrape_pubmed,
+    scrape_pubmed_article,
+)
+from amfv_datasets.scraping.rch import (
+    RchFetchError,
+    RchGuidelineRef,
+    scrape_rch,
+)
 from amfv_datasets.scraping.who import (
     WhoFetchError,
     WhoListingPage,
@@ -49,13 +75,20 @@
 )
 
 __all__ = [
-    "GuidanceRef",
+    "CpsFetchError",
+    "CpsStatementRef",
     "GuidanceListingPage",
+    "GuidanceRef",
     "IDSAFetchError",
     "IDSAGuidelineRef",
     "LinkMode",
     "NiceFetchError",
     "OutputFormat",
+    "PubMedArticleRef",
+    "PubMedFetchError",
+    "PubMedListingPage",
+    "RchFetchError",
+    "RchGuidelineRef",
     "ScrapeError",
     "ScrapeRun",
     "ScrapedDocument",
@@ -67,22 +100,34 @@
     "absolute_unique_urls",
     "build_guideline_text",
     "build_publication_text",
+    "build_pubmed_article_text",
+    "build_statement_text",
     "clean_text",
-    "document_title",
     "default_client",
+    "document_title",
     "first_matching_urls",
     "guidance_ref_from_url",
     "html_to_markdown",
     "idsa_ref_from_url",
     "list_practice_guidelines",
-    "list_published_guidance",
     "list_publications",
+    "list_published_guidance",
+    "list_pubmed_guidelines",
+    "list_statements",
+    "listing_page_url",
     "publication_ref_from_url",
+    "pubmed_ref_from_url",
+    "scrape_cps",
     "scrape_guideline",
     "scrape_idsa",
     "scrape_idsa_guideline",
     "scrape_listing_documents",
     "scrape_nice",
     "scrape_publication",
+    "scrape_pubmed",
+    "scrape_pubmed_article",
+    "scrape_rch",
+    "scrape_statement",
     "scrape_who",
+    "statement_ref_from_url",
 ]
diff --git a/datasets/amfv_datasets/scraping/cli.py b/datasets/amfv_datasets/scraping/cli.py
index 59e69bd7..1f423a52 100644
--- a/datasets/amfv_datasets/scraping/cli.py
+++ b/datasets/amfv_datasets/scraping/cli.py
@@ -28,6 +28,10 @@
 from amfv_datasets.scraping.html import LinkMode
 from amfv_datasets.scraping.nice import scrape_nice
 from amfv_datasets.scraping.who import scrape_who
+from amfv_datasets.scraping.idsa import scrape_idsa
+from amfv_datasets.scraping.cps import scrape_cps
+from amfv_datasets.scraping.rch import scrape_rch
+from amfv_datasets.scraping.pubmed import scrape_pubmed
 
 
 class ScraperSource(StrEnum):
@@ -36,6 +40,10 @@ class ScraperSource(StrEnum):
     ALL = "all"
     NICE = "nice"
     WHO = "who"
+    IDSA = "idsa"
+    CPS = "cps"
+    RCH = "rch"
+    PUBMED = "pubmed"
 
 
 class OutputFormat(StrEnum):
@@ -76,6 +84,14 @@ def scrape_documents(
                 return scrape_nice(documents=documents, link_mode=link_mode, url=url)
             case ScraperSource.WHO:
                 return scrape_who(documents=documents, link_mode=link_mode, url=url)
+            case ScraperSource.IDSA:
+                return scrape_idsa(documents=documents, link_mode=link_mode, url=url)
+            case ScraperSource.CPS:
+                return scrape_cps(documents=documents, link_mode=link_mode, url=url)
+            case ScraperSource.RCH:
+                return scrape_rch(documents=documents, link_mode=link_mode, url=url)
+            case ScraperSource.PUBMED:
+                return scrape_pubmed(documents=documents, link_mode=link_mode, url=url)
             case ScraperSource.ALL:
                 raise AssertionError("expanded source cannot be all")
     raise AssertionError(f"unsupported source: {source}")
@@ -129,7 +145,7 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path
 
 def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]:
     if source is ScraperSource.ALL:
-        return (ScraperSource.NICE, ScraperSource.WHO)
+        return (ScraperSource.NICE,)
     return (source,)
 
 

From 063d9dd235e75675242babfd8f28bbc0d28e95f2 Mon Sep 17 00:00:00 2001
From: ahoblitz <ahoblitz@gmail.com>
Date: Sat, 8 Aug 2026 10:51:10 -0700
Subject: [PATCH 11/15] register the merged scrapers on PR 8 dispatcher

---
 datasets/amfv_datasets/scraping/__init__.py |  8 ++-
 datasets/amfv_datasets/scraping/cli.py      | 64 ++++++++++++++++-----
 datasets/amfv_datasets/scraping/who.py      | 11 +---
 datasets/test/test_scraping_cps.py          | 10 +---
 datasets/test/test_scraping_who.py          |  3 +-
 5 files changed, 60 insertions(+), 36 deletions(-)

diff --git a/datasets/amfv_datasets/scraping/__init__.py b/datasets/amfv_datasets/scraping/__init__.py
index f48815b6..dce95784 100644
--- a/datasets/amfv_datasets/scraping/__init__.py
+++ b/datasets/amfv_datasets/scraping/__init__.py
@@ -1,10 +1,10 @@
 """Web scraping helpers and source-specific scrapers."""
 
 from amfv_datasets.scraping.base import (
+    USER_AGENT,
+    ScrapedDocument,
     ScrapeError,
     ScrapeRun,
-    ScrapedDocument,
-    USER_AGENT,
     default_client,
     scrape_listing_documents,
 )
@@ -35,9 +35,11 @@
     IDSAGuidelineRef,
     idsa_ref_from_url,
     list_practice_guidelines,
-    scrape_guideline as scrape_idsa_guideline,
     scrape_idsa,
 )
+from amfv_datasets.scraping.idsa import (
+    scrape_guideline as scrape_idsa_guideline,
+)
 from amfv_datasets.scraping.nice import (
     GuidanceListingPage,
     GuidanceRef,
diff --git a/datasets/amfv_datasets/scraping/cli.py b/datasets/amfv_datasets/scraping/cli.py
index 1f423a52..47545c0b 100644
--- a/datasets/amfv_datasets/scraping/cli.py
+++ b/datasets/amfv_datasets/scraping/cli.py
@@ -8,6 +8,7 @@
 from collections.abc import Iterable
 from dataclasses import asdict
 from enum import StrEnum
+from itertools import chain
 from pathlib import Path
 from typing import Annotated, TextIO
 
@@ -25,22 +26,22 @@
 )
 
 from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun
+from amfv_datasets.scraping.cps import scrape_cps
 from amfv_datasets.scraping.html import LinkMode
-from amfv_datasets.scraping.nice import scrape_nice
-from amfv_datasets.scraping.who import scrape_who
 from amfv_datasets.scraping.idsa import scrape_idsa
-from amfv_datasets.scraping.cps import scrape_cps
-from amfv_datasets.scraping.rch import scrape_rch
+from amfv_datasets.scraping.nice import scrape_nice
 from amfv_datasets.scraping.pubmed import scrape_pubmed
+from amfv_datasets.scraping.rch import scrape_rch
+from amfv_datasets.scraping.who import scrape_who
 
 
 class ScraperSource(StrEnum):
     """Supported scraper sources."""
 
     ALL = "all"
+    IDSA = "idsa"
     NICE = "nice"
     WHO = "who"
-    IDSA = "idsa"
     CPS = "cps"
     RCH = "rch"
     PUBMED = "pubmed"
@@ -63,6 +64,8 @@ def scrape_documents(
     documents: int | None,
     link_mode: LinkMode,
     url: str | None = None,
+    include_archived: bool = False,
+    include_in_development: bool = False,
 ) -> ScrapeRun:
     """Configure a scrape for a source.
 
@@ -74,27 +77,45 @@ def scrape_documents(
         link_mode: Whether links are kept as markdown links or stripped to their
             visible text.
         url: Source URL to scrape as a single document (default: None).
+        include_archived: Whether IDSA archived guidelines are included
+            (default: False).
+        include_in_development: Whether IDSA in-development guidelines are
+            included (default: False).
     """
     if documents is not None and documents < 1:
         raise ValueError(f"documents must be at least 1; got {documents}")
 
+    scrape_runs: list[ScrapeRun] = []
     for selected_source in _expand_source(source):
         match selected_source:
+            case ScraperSource.IDSA:
+                scrape_runs.append(
+                    scrape_idsa(
+                        documents=documents,
+                        link_mode=link_mode,
+                        url=url,
+                        include_archived=include_archived,
+                        include_in_development=include_in_development,
+                    )
+                )
             case ScraperSource.NICE:
-                return scrape_nice(documents=documents, link_mode=link_mode, url=url)
+                scrape_runs.append(scrape_nice(documents=documents, link_mode=link_mode, url=url))
             case ScraperSource.WHO:
-                return scrape_who(documents=documents, link_mode=link_mode, url=url)
-            case ScraperSource.IDSA:
-                return scrape_idsa(documents=documents, link_mode=link_mode, url=url)
+                scrape_runs.append(scrape_who(documents=documents, link_mode=link_mode, url=url))
             case ScraperSource.CPS:
-                return scrape_cps(documents=documents, link_mode=link_mode, url=url)
+                scrape_runs.append(scrape_cps(documents=documents, link_mode=link_mode, url=url))
             case ScraperSource.RCH:
-                return scrape_rch(documents=documents, link_mode=link_mode, url=url)
+                scrape_runs.append(scrape_rch(documents=documents, link_mode=link_mode, url=url))
             case ScraperSource.PUBMED:
-                return scrape_pubmed(documents=documents, link_mode=link_mode, url=url)
+                scrape_runs.append(scrape_pubmed(documents=documents, link_mode=link_mode, url=url))
             case ScraperSource.ALL:
                 raise AssertionError("expanded source cannot be all")
-    raise AssertionError(f"unsupported source: {source}")
+    if not scrape_runs:
+        raise AssertionError(f"unsupported source: {source}")
+    if len(scrape_runs) == 1:
+        return scrape_runs[0]
+    total = None if any(run.total is None for run in scrape_runs) else sum(run.total or 0 for run in scrape_runs)
+    return ScrapeRun(documents=chain.from_iterable(run.documents for run in scrape_runs), total=total)
 
 
 def write_jsonl(documents: Iterable[ScrapedDocument], output: TextIO) -> int:
@@ -145,7 +166,7 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path
 
 def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]:
     if source is ScraperSource.ALL:
-        return (ScraperSource.NICE,)
+        return (ScraperSource.NICE, ScraperSource.IDSA)
     return (source,)
 
 
@@ -155,6 +176,8 @@ def run(
     url: Annotated[str | None, typer.Option("--url", help="Source URL to scrape as a single document.")] = None,
     documents: Annotated[str, typer.Option("--documents", help="Number of documents to scrape, or 'all'.")] = "1",
     link_mode: Annotated[LinkMode, typer.Option("--links", help="Whether to keep markdown links or strip links to text.")] = LinkMode.KEEP,  # noqa: E501
+    include_archived: Annotated[bool, typer.Option("--include-archived/--exclude-archived", help="Include archived guidelines for sources that expose archival status.")] = False,  # noqa: E501
+    include_in_development: Annotated[bool, typer.Option("--include-in-development/--exclude-in-development", help="Include in-development guidelines for sources that expose development status.")] = False,  # noqa: E501
     output_format: Annotated[OutputFormat, typer.Option("--format", "-f", help="Output format.")] = OutputFormat.JSONL,
     output_path: Annotated[Path | None, typer.Option("--output", "-o", help="Output JSONL file, markdown directory, or Hugging Face dataset directory. JSONL defaults to stdout.")] = None,  # noqa: E501
     progress: Annotated[bool, typer.Option("--progress/--no-progress", help="Show a Rich progress bar.")] = True,
@@ -167,6 +190,10 @@ def run(
         documents: Number of documents to scrape, or "all" (default: "1").
         link_mode: Whether links are kept as markdown links or stripped to their
             visible text (default: LinkMode.KEEP).
+        include_archived: Whether archived guidelines are included for sources
+            that expose archival status (default: False).
+        include_in_development: Whether in-development guidelines are included
+            for sources that expose development status (default: False).
         output_format: Output format to write (default: OutputFormat.JSONL).
         output_path: Output JSONL file, markdown directory, or Hugging Face
             dataset directory. When unset, JSONL is written to stdout (default:
@@ -174,7 +201,14 @@ def run(
         progress: Whether to show a Rich progress bar (default: True).
     """
     parsed_documents = _parse_documents(documents)
-    scrape_run = scrape_documents(source, documents=parsed_documents, link_mode=link_mode, url=url)
+    scrape_run = scrape_documents(
+        source,
+        documents=parsed_documents,
+        link_mode=link_mode,
+        url=url,
+        include_archived=include_archived,
+        include_in_development=include_in_development,
+    )
     scraped_documents = scrape_run.documents
     if progress:
         scraped_documents = _progress_documents(scraped_documents, total=scrape_run.total)
diff --git a/datasets/amfv_datasets/scraping/who.py b/datasets/amfv_datasets/scraping/who.py
index a2412511..757232d6 100644
--- a/datasets/amfv_datasets/scraping/who.py
+++ b/datasets/amfv_datasets/scraping/who.py
@@ -57,6 +57,7 @@
 )
 _ISBN_RE = re.compile(r"ISBN:\s*([\d\-]+)", re.IGNORECASE)
 
+
 class WhoFetchError(ScrapeError):
     """Raised when a WHO publication cannot be fetched or parsed."""
 
@@ -96,9 +97,7 @@ def publication_ref_from_url(url: str) -> WhoPublicationRef:
 
     match = _PUBLICATION_PATH_RE.match(parsed.path.rstrip("/") + "/")
     if not match:
-        raise WhoFetchError(
-            f"Enter a URL like https://www.who.int/publications/i/item/9789240121805; got {url!r}"
-        )
+        raise WhoFetchError(f"Enter a URL like https://www.who.int/publications/i/item/9789240121805; got {url!r}")
 
     publication_id = match.group("publication_id")
     return WhoPublicationRef(
@@ -269,11 +268,7 @@ def build_publication_text(
     if not sections:
         raise WhoFetchError(f"No publication content section for '{ref.publication_id}'")
 
-    title = (
-        ref.title
-        if ref.title != ref.publication_id
-        else document_title(html_text, fallback=ref.publication_id)
-    )
+    title = ref.title if ref.title != ref.publication_id else document_title(html_text, fallback=ref.publication_id)
     overview_html = _overview_html(sections[0])
     markdown = html_to_markdown(overview_html, link_mode=link_mode, base_url=BASE_URL).strip()
     if not markdown:
diff --git a/datasets/test/test_scraping_cps.py b/datasets/test/test_scraping_cps.py
index 26d3bd93..c50ce14c 100644
--- a/datasets/test/test_scraping_cps.py
+++ b/datasets/test/test_scraping_cps.py
@@ -130,14 +130,8 @@ def handler(request: httpx.Request) -> httpx.Response:
     }
     assert "- Assess the infant:\n  - Check vital signs." in document.content
     assert "[risk calculator](https://cps.ca/en/tools/risk-calculator)" in document.content
-    assert (
-        "Current citation[1](https://cps.ca/en/documents/position/febrile-young-infants#ref1)."
-        in document.content
-    )
-    assert (
-        "Legacy citation[2](https://cps.ca/en/documents/position/febrile-young-infants#ref2)."
-        in document.content
-    )
+    assert "Current citation[1](https://cps.ca/en/documents/position/febrile-young-infants#ref1)." in document.content
+    assert "Legacy citation[2](https://cps.ca/en/documents/position/febrile-young-infants#ref2)." in document.content
     assert "38<sup>o</sup>C" in document.content
     assert "10<sup>9</sup>/L" in document.content
     assert "# Management of well-appearing" not in document.content
diff --git a/datasets/test/test_scraping_who.py b/datasets/test/test_scraping_who.py
index d3db37b3..110c5a56 100644
--- a/datasets/test/test_scraping_who.py
+++ b/datasets/test/test_scraping_who.py
@@ -25,8 +25,7 @@
     "associated with the use of intravascular catheters: part 2: central venous catheters"
 )
 _CERVICAL_GUIDELINE_TITLE = (
-    "WHO guideline for screening and treatment of cervical pre-cancer lesions "
-    "for cervical cancer prevention"
+    "WHO guideline for screening and treatment of cervical pre-cancer lesions for cervical cancer prevention"
 )
 
 

From 22250867ae3a5449be5061088c115aadc5640b4b Mon Sep 17 00:00:00 2001
From: Zander Giuffrida <zndr27@gmail.com>
Date: Mon, 17 Aug 2026 23:35:34 -0700
Subject: [PATCH 12/15] drop the vendored scrapers and their tests

Leaves only the base.py and html.py changes the source branches share.
Each source PR adds its own module, tests and licence notes when it merges.
---
 .../amfv_datasets/scraping/LICENSE_NOTES.md   |  30 -
 datasets/amfv_datasets/scraping/__init__.py   |  87 +--
 datasets/amfv_datasets/scraping/cli.py        |  62 +-
 datasets/amfv_datasets/scraping/cps.py        | 285 -------
 datasets/amfv_datasets/scraping/idsa.py       | 385 ----------
 datasets/amfv_datasets/scraping/pubmed.py     | 723 ------------------
 datasets/amfv_datasets/scraping/rch.py        | 398 ----------
 datasets/amfv_datasets/scraping/who.py        | 374 ---------
 datasets/test/fixtures/who_listing_api.json   |  20 -
 .../fixtures/who_publication_overview.html    |  17 -
 datasets/test/test_scraping_cli.py            | 132 ----
 datasets/test/test_scraping_cps.py            | 153 ----
 datasets/test/test_scraping_idsa.py           | 258 -------
 datasets/test/test_scraping_pubmed.py         | 478 ------------
 datasets/test/test_scraping_rch.py            | 252 ------
 datasets/test/test_scraping_who.py            | 154 ----
 16 files changed, 7 insertions(+), 3801 deletions(-)
 delete mode 100644 datasets/amfv_datasets/scraping/LICENSE_NOTES.md
 delete mode 100644 datasets/amfv_datasets/scraping/cps.py
 delete mode 100644 datasets/amfv_datasets/scraping/idsa.py
 delete mode 100644 datasets/amfv_datasets/scraping/pubmed.py
 delete mode 100644 datasets/amfv_datasets/scraping/rch.py
 delete mode 100644 datasets/amfv_datasets/scraping/who.py
 delete mode 100644 datasets/test/fixtures/who_listing_api.json
 delete mode 100644 datasets/test/fixtures/who_publication_overview.html
 delete mode 100644 datasets/test/test_scraping_cps.py
 delete mode 100644 datasets/test/test_scraping_idsa.py
 delete mode 100644 datasets/test/test_scraping_pubmed.py
 delete mode 100644 datasets/test/test_scraping_rch.py
 delete mode 100644 datasets/test/test_scraping_who.py

diff --git a/datasets/amfv_datasets/scraping/LICENSE_NOTES.md b/datasets/amfv_datasets/scraping/LICENSE_NOTES.md
deleted file mode 100644
index a98b822b..00000000
--- a/datasets/amfv_datasets/scraping/LICENSE_NOTES.md
+++ /dev/null
@@ -1,30 +0,0 @@
-# Source licensing notes for scraped corpora
-
-## WHO (World Health Organization)
-
-Publications on [who.int](https://www.who.int/publications) published since
-November 2016 are licensed under **Creative Commons Attribution-NonCommercial-
-ShareAlike 3.0 IGO** (CC BY-NC-SA 3.0 IGO).
-
-- **Non-commercial use and adaptation** are permitted.
-- **Attribution** to WHO is required.
-- **Share-alike**: derivatives must use the same or a similar licence.
-
-Pre-2017 publications were not reissued under this licence. Use
-`metadata.publication_date` to filter when building corpora.
-
-### Suggested attribution
-
-> © World Health Organization {year}. *{publication title}*.
-> Licensed under CC BY-NC-SA 3.0 IGO.
-> https://creativecommons.org/licenses/by-nc-sa/3.0/igo/
-
-Each scraped document also records `metadata.license` and
-`metadata.attribution`.
-
-### Milestone 1 content scope
-
-The WHO scraper collects the HTML **Overview** section from each publication
-landing page (`metadata.content_scope = "overview"`). Full guideline text is
-typically available only as a linked PDF (`metadata.download_url`). PDF
-extraction is intentionally deferred.
diff --git a/datasets/amfv_datasets/scraping/__init__.py b/datasets/amfv_datasets/scraping/__init__.py
index dce95784..cd7cdb06 100644
--- a/datasets/amfv_datasets/scraping/__init__.py
+++ b/datasets/amfv_datasets/scraping/__init__.py
@@ -8,20 +8,7 @@
     default_client,
     scrape_listing_documents,
 )
-from amfv_datasets.scraping.cli import (
-    OutputFormat,
-    ScraperSource,
-)
-from amfv_datasets.scraping.cps import (
-    CpsFetchError,
-    CpsStatementRef,
-    build_statement_text,
-    list_statements,
-    listing_page_url,
-    scrape_cps,
-    scrape_statement,
-    statement_ref_from_url,
-)
+from amfv_datasets.scraping.cli import OutputFormat, ScraperSource
 from amfv_datasets.scraping.html import (
     LinkMode,
     absolute_unique_urls,
@@ -30,16 +17,6 @@
     first_matching_urls,
     html_to_markdown,
 )
-from amfv_datasets.scraping.idsa import (
-    IDSAFetchError,
-    IDSAGuidelineRef,
-    idsa_ref_from_url,
-    list_practice_guidelines,
-    scrape_idsa,
-)
-from amfv_datasets.scraping.idsa import (
-    scrape_guideline as scrape_idsa_guideline,
-)
 from amfv_datasets.scraping.nice import (
     GuidanceListingPage,
     GuidanceRef,
@@ -50,86 +27,28 @@
     scrape_guideline,
     scrape_nice,
 )
-from amfv_datasets.scraping.pubmed import (
-    PubMedArticleRef,
-    PubMedFetchError,
-    PubMedListingPage,
-    build_pubmed_article_text,
-    list_pubmed_guidelines,
-    pubmed_ref_from_url,
-    scrape_pubmed,
-    scrape_pubmed_article,
-)
-from amfv_datasets.scraping.rch import (
-    RchFetchError,
-    RchGuidelineRef,
-    scrape_rch,
-)
-from amfv_datasets.scraping.who import (
-    WhoFetchError,
-    WhoListingPage,
-    WhoPublicationRef,
-    build_publication_text,
-    list_publications,
-    publication_ref_from_url,
-    scrape_publication,
-    scrape_who,
-)
 
 __all__ = [
-    "CpsFetchError",
-    "CpsStatementRef",
-    "GuidanceListingPage",
     "GuidanceRef",
-    "IDSAFetchError",
-    "IDSAGuidelineRef",
+    "GuidanceListingPage",
     "LinkMode",
     "NiceFetchError",
     "OutputFormat",
-    "PubMedArticleRef",
-    "PubMedFetchError",
-    "PubMedListingPage",
-    "RchFetchError",
-    "RchGuidelineRef",
     "ScrapeError",
     "ScrapeRun",
     "ScrapedDocument",
     "ScraperSource",
     "USER_AGENT",
-    "WhoFetchError",
-    "WhoListingPage",
-    "WhoPublicationRef",
     "absolute_unique_urls",
     "build_guideline_text",
-    "build_publication_text",
-    "build_pubmed_article_text",
-    "build_statement_text",
     "clean_text",
-    "default_client",
     "document_title",
+    "default_client",
     "first_matching_urls",
     "guidance_ref_from_url",
     "html_to_markdown",
-    "idsa_ref_from_url",
-    "list_practice_guidelines",
-    "list_publications",
     "list_published_guidance",
-    "list_pubmed_guidelines",
-    "list_statements",
-    "listing_page_url",
-    "publication_ref_from_url",
-    "pubmed_ref_from_url",
-    "scrape_cps",
     "scrape_guideline",
-    "scrape_idsa",
-    "scrape_idsa_guideline",
     "scrape_listing_documents",
     "scrape_nice",
-    "scrape_publication",
-    "scrape_pubmed",
-    "scrape_pubmed_article",
-    "scrape_rch",
-    "scrape_statement",
-    "scrape_who",
-    "statement_ref_from_url",
 ]
diff --git a/datasets/amfv_datasets/scraping/cli.py b/datasets/amfv_datasets/scraping/cli.py
index 47545c0b..e8aece34 100644
--- a/datasets/amfv_datasets/scraping/cli.py
+++ b/datasets/amfv_datasets/scraping/cli.py
@@ -8,7 +8,6 @@
 from collections.abc import Iterable
 from dataclasses import asdict
 from enum import StrEnum
-from itertools import chain
 from pathlib import Path
 from typing import Annotated, TextIO
 
@@ -26,25 +25,15 @@
 )
 
 from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun
-from amfv_datasets.scraping.cps import scrape_cps
 from amfv_datasets.scraping.html import LinkMode
-from amfv_datasets.scraping.idsa import scrape_idsa
 from amfv_datasets.scraping.nice import scrape_nice
-from amfv_datasets.scraping.pubmed import scrape_pubmed
-from amfv_datasets.scraping.rch import scrape_rch
-from amfv_datasets.scraping.who import scrape_who
 
 
 class ScraperSource(StrEnum):
     """Supported scraper sources."""
 
     ALL = "all"
-    IDSA = "idsa"
     NICE = "nice"
-    WHO = "who"
-    CPS = "cps"
-    RCH = "rch"
-    PUBMED = "pubmed"
 
 
 class OutputFormat(StrEnum):
@@ -64,8 +53,6 @@ def scrape_documents(
     documents: int | None,
     link_mode: LinkMode,
     url: str | None = None,
-    include_archived: bool = False,
-    include_in_development: bool = False,
 ) -> ScrapeRun:
     """Configure a scrape for a source.
 
@@ -77,45 +64,17 @@ def scrape_documents(
         link_mode: Whether links are kept as markdown links or stripped to their
             visible text.
         url: Source URL to scrape as a single document (default: None).
-        include_archived: Whether IDSA archived guidelines are included
-            (default: False).
-        include_in_development: Whether IDSA in-development guidelines are
-            included (default: False).
     """
     if documents is not None and documents < 1:
         raise ValueError(f"documents must be at least 1; got {documents}")
 
-    scrape_runs: list[ScrapeRun] = []
     for selected_source in _expand_source(source):
         match selected_source:
-            case ScraperSource.IDSA:
-                scrape_runs.append(
-                    scrape_idsa(
-                        documents=documents,
-                        link_mode=link_mode,
-                        url=url,
-                        include_archived=include_archived,
-                        include_in_development=include_in_development,
-                    )
-                )
             case ScraperSource.NICE:
-                scrape_runs.append(scrape_nice(documents=documents, link_mode=link_mode, url=url))
-            case ScraperSource.WHO:
-                scrape_runs.append(scrape_who(documents=documents, link_mode=link_mode, url=url))
-            case ScraperSource.CPS:
-                scrape_runs.append(scrape_cps(documents=documents, link_mode=link_mode, url=url))
-            case ScraperSource.RCH:
-                scrape_runs.append(scrape_rch(documents=documents, link_mode=link_mode, url=url))
-            case ScraperSource.PUBMED:
-                scrape_runs.append(scrape_pubmed(documents=documents, link_mode=link_mode, url=url))
+                return scrape_nice(documents=documents, link_mode=link_mode, url=url)
             case ScraperSource.ALL:
                 raise AssertionError("expanded source cannot be all")
-    if not scrape_runs:
-        raise AssertionError(f"unsupported source: {source}")
-    if len(scrape_runs) == 1:
-        return scrape_runs[0]
-    total = None if any(run.total is None for run in scrape_runs) else sum(run.total or 0 for run in scrape_runs)
-    return ScrapeRun(documents=chain.from_iterable(run.documents for run in scrape_runs), total=total)
+    raise AssertionError(f"unsupported source: {source}")
 
 
 def write_jsonl(documents: Iterable[ScrapedDocument], output: TextIO) -> int:
@@ -166,7 +125,7 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path
 
 def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]:
     if source is ScraperSource.ALL:
-        return (ScraperSource.NICE, ScraperSource.IDSA)
+        return (ScraperSource.NICE,)
     return (source,)
 
 
@@ -176,8 +135,6 @@ def run(
     url: Annotated[str | None, typer.Option("--url", help="Source URL to scrape as a single document.")] = None,
     documents: Annotated[str, typer.Option("--documents", help="Number of documents to scrape, or 'all'.")] = "1",
     link_mode: Annotated[LinkMode, typer.Option("--links", help="Whether to keep markdown links or strip links to text.")] = LinkMode.KEEP,  # noqa: E501
-    include_archived: Annotated[bool, typer.Option("--include-archived/--exclude-archived", help="Include archived guidelines for sources that expose archival status.")] = False,  # noqa: E501
-    include_in_development: Annotated[bool, typer.Option("--include-in-development/--exclude-in-development", help="Include in-development guidelines for sources that expose development status.")] = False,  # noqa: E501
     output_format: Annotated[OutputFormat, typer.Option("--format", "-f", help="Output format.")] = OutputFormat.JSONL,
     output_path: Annotated[Path | None, typer.Option("--output", "-o", help="Output JSONL file, markdown directory, or Hugging Face dataset directory. JSONL defaults to stdout.")] = None,  # noqa: E501
     progress: Annotated[bool, typer.Option("--progress/--no-progress", help="Show a Rich progress bar.")] = True,
@@ -190,10 +147,6 @@ def run(
         documents: Number of documents to scrape, or "all" (default: "1").
         link_mode: Whether links are kept as markdown links or stripped to their
             visible text (default: LinkMode.KEEP).
-        include_archived: Whether archived guidelines are included for sources
-            that expose archival status (default: False).
-        include_in_development: Whether in-development guidelines are included
-            for sources that expose development status (default: False).
         output_format: Output format to write (default: OutputFormat.JSONL).
         output_path: Output JSONL file, markdown directory, or Hugging Face
             dataset directory. When unset, JSONL is written to stdout (default:
@@ -201,14 +154,7 @@ def run(
         progress: Whether to show a Rich progress bar (default: True).
     """
     parsed_documents = _parse_documents(documents)
-    scrape_run = scrape_documents(
-        source,
-        documents=parsed_documents,
-        link_mode=link_mode,
-        url=url,
-        include_archived=include_archived,
-        include_in_development=include_in_development,
-    )
+    scrape_run = scrape_documents(source, documents=parsed_documents, link_mode=link_mode, url=url)
     scraped_documents = scrape_run.documents
     if progress:
         scraped_documents = _progress_documents(scraped_documents, total=scrape_run.total)
diff --git a/datasets/amfv_datasets/scraping/cps.py b/datasets/amfv_datasets/scraping/cps.py
deleted file mode 100644
index 4b9ba1e9..00000000
--- a/datasets/amfv_datasets/scraping/cps.py
+++ /dev/null
@@ -1,285 +0,0 @@
-"""Scrape Canadian Paediatric Society statements into normalized markdown.
-
-CPS retains copyright in its position statements and practice points. This
-module supports a permission-gated ingestion workflow; obtain written CPS
-permission before running a corpus scrape or redistributing its output.
-"""
-
-from __future__ import annotations
-
-import re
-from collections.abc import Iterable
-from dataclasses import dataclass
-from urllib.parse import unquote, urljoin, urlparse
-
-import httpx
-from lxml import html as lxml_html
-from markdownify import MarkdownConverter
-
-from amfv_datasets.scraping.base import (
-    ScrapedDocument,
-    ScrapeError,
-    ScrapeRun,
-    default_client,
-    scrape_listing_documents,
-)
-from amfv_datasets.scraping.html import LinkMode, clean_text
-
-BASE_URL = "https://cps.ca"
-STATEMENTS_URL = f"{BASE_URL}/en/documents/statements-by-date"
-DOCUMENT_DELAY_SECONDS = 10.0
-LISTING_PAGE_SIZE = 10
-
-_POSITION_PATH_RE = re.compile(r"^/(?:en/)?documents/position/(?P<slug>[^/]+)/?$", re.IGNORECASE)
-_DATE_PATTERNS = {
-    "posted": re.compile(r"\bPosted:\s*([A-Za-z]+\s+\d{1,2},\s+\d{4})", re.IGNORECASE),
-    "reaffirmed": re.compile(r"\bReaffirmed:\s*([A-Za-z]+\s+\d{1,2},\s+\d{4})", re.IGNORECASE),
-    "updated": re.compile(r"\bUpdated:\s*([A-Za-z]+\s+\d{1,2},\s+\d{4})", re.IGNORECASE),
-}
-_BLANK_LINES_RE = re.compile(r"\n{3,}")
-_EMPTY_MARKDOWN_LINK_RE = re.compile(r"(?<!!)\[\]\([^)]*\)")
-_DECORATIVE_IMAGE_FILENAMES = {"file-pdf.svg"}
-
-
-class _CpsMarkdownConverter(MarkdownConverter):
-    """Preserve CPS superscripts that carry clinical meaning or citations."""
-
-    def convert_a(self, element, text: str, parent_tags: set[str]) -> str:  # noqa: ANN001
-        classes = element.get("class") or ()
-        href = element.get("href")
-        if "reference" in classes and href and "#ref" in href:
-            marker = element.get_text(strip=True).strip("[]")
-            if "a" in (self.options.get("strip") or ()):
-                return f"[{marker}]"
-            return f"[{marker}]({href})"
-        return super().convert_a(element, text, parent_tags)
-
-    def convert_sup(self, element, text: str, parent_tags: set[str]) -> str:  # noqa: ANN001
-        if not text.strip():
-            return ""
-        citation = element.find("a", href=lambda href: href and "#ref" in href)
-        if citation:
-            marker = element.get_text(strip=True).strip("[]")
-            if "a" in (self.options.get("strip") or ()):
-                return f"[{marker}]"
-            return f"[{marker}]({citation.get('href')})"
-        return f"<sup>{text}</sup>"
-
-
-_CPS_MARKDOWN_CONVERTERS = {
-    LinkMode.KEEP: _CpsMarkdownConverter(bullets="-", heading_style="ATX"),
-    LinkMode.STRIP: _CpsMarkdownConverter(bullets="-", heading_style="ATX", strip=("a",)),
-}
-
-
-class CpsFetchError(ScrapeError):
-    """Raised when a CPS statement cannot be discovered or parsed."""
-
-
-@dataclass(frozen=True)
-class CpsStatementRef:
-    """A CPS statement or practice point discovered from the date index."""
-
-    slug: str
-    title: str
-    page_url: str
-
-
-def statement_ref_from_url(url: str, *, title: str | None = None) -> CpsStatementRef:
-    """Parse a CPS position-statement URL into a canonical reference."""
-    parsed = urlparse(url.strip())
-    if parsed.scheme not in {"http", "https"} or parsed.netloc.lower() not in {"cps.ca", "www.cps.ca"}:
-        raise CpsFetchError(f"Enter a CPS statement URL from cps.ca; got {url!r}")
-
-    match = _POSITION_PATH_RE.match(unquote(parsed.path))
-    if not match:
-        raise CpsFetchError(f"Enter a URL like https://cps.ca/en/documents/position/example-statement; got {url!r}")
-    slug = match.group("slug")
-    return CpsStatementRef(
-        slug=slug,
-        title=title or slug.replace("-", " "),
-        page_url=f"{BASE_URL}/en/documents/position/{slug}",
-    )
-
-
-def listing_page_url(page: int) -> str:
-    """Return the CPS date-index URL for a one-based page number."""
-    if page < 1:
-        raise ValueError(f"page must be at least 1; got {page}")
-    if page == 1:
-        return STATEMENTS_URL
-    return f"{STATEMENTS_URL}/P{(page - 1) * LISTING_PAGE_SIZE}"
-
-
-def list_statements(client: httpx.Client, page: int) -> list[CpsStatementRef]:
-    """Return unique statement references from one CPS date-index page."""
-    response = client.get(listing_page_url(page))
-    response.raise_for_status()
-    doc = lxml_html.fromstring(response.text)
-    anchors = doc.xpath("//div[contains(concat(' ', normalize-space(@class), ' '), ' stmt-title ')]//a[@href]")
-
-    refs: list[CpsStatementRef] = []
-    seen_urls: set[str] = set()
-    for anchor in anchors:
-        href = anchor.get("href")
-        title = clean_text(anchor.text_content(), drop_numeric_citations=False)
-        if not href or not title:
-            continue
-        try:
-            ref = statement_ref_from_url(urljoin(BASE_URL, href), title=title)
-        except CpsFetchError:
-            continue
-        if ref.page_url in seen_urls:
-            continue
-        seen_urls.add(ref.page_url)
-        refs.append(ref)
-    return refs
-
-
-def _content_root(html_text: str) -> lxml_html.HtmlElement:
-    doc = lxml_html.fromstring(html_text)
-    selectors = (
-        "//div[contains(concat(' ', normalize-space(@class), ' '), ' statement-wrapper ')][1]",
-        "//main//article[1]",
-        "//*[@id='main-content']//article[1]",
-        "//*[@id='main-content'][1]",
-        "//main[1]",
-    )
-    for selector in selectors:
-        matches = doc.xpath(selector)
-        if matches:
-            return matches[0]
-    raise CpsFetchError("No readable statement content found on the CPS page")
-
-
-def _normalize_content(root: lxml_html.HtmlElement, *, base_url: str) -> None:
-    removable = root.xpath(
-        ".//script | .//style | .//nav | .//form | .//button | .//noscript | "
-        ".//*[contains(concat(' ', normalize-space(@class), ' '), ' breadcrumb ')] | "
-        ".//*[contains(@class, 'share')] | "
-        ".//*[contains(@class, 'social')] | "
-        ".//*[contains(concat(' ', normalize-space(@class), ' '), ' related-content ')] | "
-        ".//*[contains(concat(' ', normalize-space(@class), ' '), ' sidebar ')] | "
-        ".//*[contains(concat(' ', normalize-space(@class), ' '), ' hide-for-print ')] | "
-        ".//*[contains(concat(' ', normalize-space(@class), ' '), ' show-for-print ')] | "
-        ".//*[contains(@class, 'print:tw-hidden')] | "
-        ".//*[contains(concat(' ', normalize-space(@class), ' '), ' --podcast ')]"
-    )
-    for element in removable:
-        element.drop_tree()
-
-    for link in root.xpath(".//a[contains(@href, '/en/education/test-your-knowledge')]"):
-        link.drop_tree()
-
-    for image in root.xpath(".//img[@src]"):
-        filename = urlparse(image.get("src")).path.rsplit("/", 1)[-1].lower()
-        if filename in _DECORATIVE_IMAGE_FILENAMES:
-            image.drop_tree()
-
-    for link in root.xpath(".//a[@href]"):
-        link.set("href", urljoin(base_url, link.get("href")))
-    for image in root.xpath(".//img[@src]"):
-        image.set("src", urljoin(base_url, image.get("src")))
-
-    for nested in root.xpath(".//strong//strong | .//em//em"):
-        nested.drop_tag()
-    for title in root.xpath(".//h1"):
-        title.drop_tree()
-    for heading in root.xpath(".//*[self::h1 or self::h2 or self::h3 or self::h4 or self::h5 or self::h6]"):
-        for emphasis in heading.xpath(".//strong | .//em | .//b | .//i"):
-            emphasis.drop_tag()
-        if not clean_text(heading.text_content(), drop_numeric_citations=False) and not heading.xpath(".//img"):
-            heading.drop_tree()
-
-
-def _content_to_markdown(root: lxml_html.HtmlElement, *, link_mode: LinkMode) -> str:
-    source = lxml_html.tostring(root, encoding="unicode")
-    markdown = _CPS_MARKDOWN_CONVERTERS[link_mode].convert(source)
-    markdown = _EMPTY_MARKDOWN_LINK_RE.sub("", markdown)
-    lines = [line.rstrip() for line in markdown.splitlines()]
-    return _BLANK_LINES_RE.sub("\n\n", "\n".join(lines)).strip()
-
-
-def build_statement_text(
-    html_text: str,
-    *,
-    link_mode: LinkMode = LinkMode.KEEP,
-    base_url: str = BASE_URL,
-) -> tuple[str, int]:
-    """Extract one CPS statement as markdown and return its section count."""
-    root = _content_root(html_text)
-    _normalize_content(root, base_url=base_url)
-    section_count = max(1, len(root.xpath(".//*[self::h2 or self::h3 or self::h4 or self::h5 or self::h6]")))
-    content = _content_to_markdown(root, link_mode=link_mode)
-    if not content:
-        raise CpsFetchError("No readable statement content found on the CPS page")
-    return content, section_count
-
-
-def scrape_statement(
-    client: httpx.Client,
-    ref: CpsStatementRef,
-    *,
-    link_mode: LinkMode = LinkMode.KEEP,
-) -> ScrapedDocument:
-    """Scrape one CPS statement into the shared document schema."""
-    response = client.get(ref.page_url)
-    response.raise_for_status()
-    content, section_count = build_statement_text(response.text, link_mode=link_mode, base_url=ref.page_url)
-    raw_root = _content_root(response.text)
-    headings = [clean_text(value) for value in raw_root.xpath(".//h1[1]//text()")]
-    title = " ".join(value for value in headings if value) or ref.title
-    metadata: dict[str, str] = {"slug": ref.slug}
-    visible_text = clean_text(raw_root.text_content(), drop_numeric_citations=False)
-    for key, pattern in _DATE_PATTERNS.items():
-        if match := pattern.search(visible_text):
-            metadata[key] = match.group(1).strip()
-    return ScrapedDocument(
-        source="cps",
-        external_id=f"cps-{ref.slug.lower()}",
-        title=title,
-        url=ref.page_url,
-        content=content,
-        section_count=section_count,
-        metadata=metadata,
-    )
-
-
-def scrape_cps(
-    *,
-    documents: int | None,
-    link_mode: LinkMode = LinkMode.KEEP,
-    url: str | None = None,
-) -> ScrapeRun:
-    """Configure a CPS scrape from one URL or the current-statements index."""
-    if url is not None:
-
-        def scrape_url() -> Iterable[ScrapedDocument]:
-            with default_client() as client:
-                yield scrape_statement(client, statement_ref_from_url(url), link_mode=link_mode)
-
-        return ScrapeRun(documents=scrape_url(), total=1)
-
-    return ScrapeRun(
-        documents=scrape_listing_documents(
-            documents=documents,
-            client_factory=default_client,
-            list_page=list_statements,
-            scrape_item=lambda client, ref: scrape_statement(client, ref, link_mode=link_mode),
-            document_delay_seconds=DOCUMENT_DELAY_SECONDS,
-        )
-    )
-
-
-__all__ = [
-    "BASE_URL",
-    "CpsFetchError",
-    "CpsStatementRef",
-    "STATEMENTS_URL",
-    "build_statement_text",
-    "list_statements",
-    "listing_page_url",
-    "scrape_cps",
-    "scrape_statement",
-    "statement_ref_from_url",
-]
diff --git a/datasets/amfv_datasets/scraping/idsa.py b/datasets/amfv_datasets/scraping/idsa.py
deleted file mode 100644
index 1feffcc5..00000000
--- a/datasets/amfv_datasets/scraping/idsa.py
+++ /dev/null
@@ -1,385 +0,0 @@
-"""Scrape IDSA practice guidelines into normalized markdown documents."""
-
-from __future__ import annotations
-
-import re
-from collections.abc import Iterable
-from dataclasses import dataclass
-from urllib.parse import urljoin, urlparse
-
-import httpx
-from lxml import html as lxml_html
-
-from amfv_datasets.scraping.base import (
-    ScrapedDocument,
-    ScrapeError,
-    ScrapeRun,
-    default_client,
-    scrape_listing_documents,
-)
-from amfv_datasets.scraping.html import LinkMode, absolute_unique_urls, clean_text, document_title, html_to_markdown
-
-BASE_URL = "https://www.idsociety.org"
-LISTING_URL = f"{BASE_URL}/practice-guideline/all-practice-guidelines"
-DOCUMENT_DELAY_SECONDS = 5.0
-_SHORT_CONTENT_CHARS = 10_000
-
-_IDSA_PATH_RE = re.compile(r"^/practice-guideline/(?P<slug>[^/]+)(?:/|$)", re.IGNORECASE)
-_BACK_TO_TOP_RE = re.compile(r"^\s*back to top\s*$", re.IGNORECASE)
-_TABLE_OF_CONTENTS_RE = re.compile(r"^\s*table\s+of\s+contents\s*$", re.IGNORECASE)
-_PDF_TEXT_RE = re.compile(r"\b(download\s+)?pdf\b", re.IGNORECASE)
-_WHITESPACE_RE = re.compile(r"\s+")
-
-
-class IDSAFetchError(ScrapeError):
-    """Raised when a practice guideline cannot be sourced from IDSA."""
-
-
-@dataclass(frozen=True)
-class IDSAGuidelineRef:
-    """An IDSA practice guideline reference from the A-Z listing."""
-
-    title: str
-    slug: str
-    page_url: str
-    year: int | None
-    statuses: tuple[str, ...]
-
-
-@dataclass(frozen=True)
-class IDSAGuidelineListingPage:
-    """An IDSA practice guideline listing page."""
-
-    refs: list[IDSAGuidelineRef]
-    total: int | None
-
-
-def idsa_ref_from_url(url: str) -> IDSAGuidelineRef:
-    """Parse an IDSA practice guideline URL into a canonical guideline reference.
-
-    Args:
-        url: IDSA practice guideline URL to parse.
-    """
-    parsed = urlparse(url.strip())
-    if parsed.scheme not in {"http", "https"} or parsed.netloc.lower() not in {
-        "www.idsociety.org",
-        "idsociety.org",
-    }:
-        raise IDSAFetchError(f"Enter an IDSA practice guideline URL from idsociety.org; got {url!r}")
-
-    match = _IDSA_PATH_RE.match(parsed.path.rstrip("/") + "/")
-    if not match:
-        raise IDSAFetchError(f"Enter a URL like https://www.idsociety.org/practice-guideline/example/; got {url!r}")
-
-    slug = match.group("slug").lower()
-    return IDSAGuidelineRef(
-        title=slug.replace("-", " ").title(),
-        slug=slug,
-        page_url=_page_url(slug),
-        year=None,
-        statuses=(),
-    )
-
-
-def list_practice_guidelines(
-    client: httpx.Client,
-    *,
-    include_archived: bool = False,
-    include_in_development: bool = False,
-) -> IDSAGuidelineListingPage:
-    """Return IDSA practice guideline refs from the A-Z listing.
-
-    Args:
-        client: HTTP client used to fetch the listing page.
-        include_archived: Whether archived guidelines are included (default: False).
-        include_in_development: Whether in-development guidelines are included (default: False).
-    """
-    response = client.get(LISTING_URL)
-    response.raise_for_status()
-    refs = _parse_listing(
-        response.text,
-        include_archived=include_archived,
-        include_in_development=include_in_development,
-    )
-    return IDSAGuidelineListingPage(refs=refs, total=len(refs))
-
-
-def _build_guideline_text(
-    client: httpx.Client,
-    ref: IDSAGuidelineRef,
-    *,
-    link_mode: LinkMode = LinkMode.KEEP,
-) -> tuple[str, int, str, dict[str, list[str]]]:
-    response = client.get(ref.page_url)
-    response.raise_for_status()
-    title = document_title(response.text, fallback=ref.title)
-    content_html = _guideline_content_html(response.text)
-    content = html_to_markdown(content_html, link_mode=link_mode, base_url=BASE_URL)
-    if not content:
-        raise IDSAFetchError(f"No readable content for IDSA guideline '{ref.slug}'")
-    return content, _section_count(content_html), title, _links_metadata(content_html)
-
-
-def scrape_guideline(
-    client: httpx.Client,
-    ref: IDSAGuidelineRef,
-    *,
-    link_mode: LinkMode = LinkMode.KEEP,
-) -> ScrapedDocument:
-    """Scrape an IDSA practice guideline into a normalized document.
-
-    Args:
-        client: HTTP client used to fetch the guideline page.
-        ref: IDSA guideline reference to scrape.
-        link_mode: Whether links are kept as markdown links or stripped to their
-            visible text (default: LinkMode.KEEP).
-    """
-    content, section_count, title, links_metadata = _build_guideline_text(client, ref, link_mode=link_mode)
-    return ScrapedDocument(
-        source="idsa",
-        external_id=f"idsa-{ref.slug}",
-        title=title,
-        url=ref.page_url,
-        content=content,
-        section_count=section_count,
-        metadata={
-            "year": ref.year,
-            "statuses": list(ref.statuses),
-            "slug": ref.slug,
-            "listing_url": LISTING_URL,
-            "content_length_chars": len(content),
-            "quality_flags": _quality_flags(content),
-            **links_metadata,
-        },
-    )
-
-
-def scrape_idsa(
-    *,
-    documents: int | None,
-    link_mode: LinkMode = LinkMode.KEEP,
-    url: str | None = None,
-    include_archived: bool = False,
-    include_in_development: bool = False,
-) -> ScrapeRun:
-    """Scrape IDSA practice guidelines from a URL or the A-Z listing.
-
-    Args:
-        documents: Number of documents to scrape. Ignored when `url` is set.
-            When unset, every included IDSA listing item is scraped (default:
-            None).
-        link_mode: Whether links are kept as markdown links or stripped to their
-            visible text (default: LinkMode.KEEP).
-        url: IDSA source URL to scrape as a single document (default: None).
-        include_archived: Whether archived guidelines are included (default: False).
-        include_in_development: Whether in-development guidelines are included
-            (default: False).
-    """
-    if url is not None:
-
-        def scrape_url() -> Iterable[ScrapedDocument]:
-            with default_client() as client:
-                yield scrape_guideline(client, idsa_ref_from_url(url), link_mode=link_mode)
-
-        return ScrapeRun(documents=scrape_url(), total=1)
-
-    with default_client() as client:
-        listing = list_practice_guidelines(
-            client,
-            include_archived=include_archived,
-            include_in_development=include_in_development,
-        )
-    total = listing.total if documents is None or listing.total is None else min(documents, listing.total)
-    return ScrapeRun(
-        total=total,
-        documents=scrape_listing_documents(
-            documents=documents,
-            client_factory=default_client,
-            first_page_items=listing.refs,
-            list_page=lambda _client, _page: (),
-            scrape_item=lambda client, ref: scrape_guideline(client, ref, link_mode=link_mode),
-            document_delay_seconds=DOCUMENT_DELAY_SECONDS,
-        ),
-    )
-
-
-def _parse_listing(
-    html_text: str,
-    *,
-    include_archived: bool,
-    include_in_development: bool,
-) -> list[IDSAGuidelineRef]:
-    doc = lxml_html.fromstring(html_text)
-    items = doc.xpath(
-        "//div[contains(concat(' ', normalize-space(@class), ' '), ' alpha-listing ')]"
-        "//li[.//a[contains(concat(' ', normalize-space(@class), ' '), ' list-pages__link ')]]"
-    )
-    refs: list[IDSAGuidelineRef] = []
-    for item in items:
-        link = item.xpath(".//a[contains(concat(' ', normalize-space(@class), ' '), ' list-pages__link ')][1]")
-        if not link:
-            continue
-        statuses = tuple(
-            clean_text(status, drop_numeric_citations=False)
-            for status in item.xpath(
-                ".//*[contains(concat(' ', normalize-space(@class), ' '), ' category-dot ')]/text()"
-            )
-        )
-        if not _included_statuses(
-            statuses,
-            include_archived=include_archived,
-            include_in_development=include_in_development,
-        ):
-            continue
-        href = link[0].get("href")
-        if not href:
-            continue
-        page_url = urljoin(BASE_URL, href).split("#")[0].split("?")[0]
-        match = _IDSA_PATH_RE.match(urlparse(page_url).path.rstrip("/") + "/")
-        if not match:
-            continue
-        refs.append(
-            IDSAGuidelineRef(
-                title=clean_text(link[0].text_content(), drop_numeric_citations=False),
-                slug=match.group("slug").lower(),
-                page_url=page_url,
-                year=_parse_year(item),
-                statuses=statuses,
-            )
-        )
-    return refs
-
-
-def _included_statuses(
-    statuses: tuple[str, ...],
-    *,
-    include_archived: bool,
-    include_in_development: bool,
-) -> bool:
-    has_current = "Current" in statuses
-    has_archived = "Archived" in statuses
-    has_in_development = "In Development" in statuses
-    if has_archived and not include_archived:
-        return False
-    if has_in_development and not include_in_development:
-        return False
-    return has_current or (has_archived and include_archived) or (has_in_development and include_in_development)
-
-
-def _parse_year(item: lxml_html.HtmlElement) -> int | None:
-    values = item.xpath(".//*[contains(concat(' ', normalize-space(@class), ' '), ' list-pages__year ')]/text()")
-    if not values:
-        return None
-    value = clean_text(values[0], drop_numeric_citations=False)
-    return int(value) if value.isdigit() else None
-
-
-def _guideline_content_html(html_text: str) -> str:
-    doc = lxml_html.fromstring(html_text)
-    candidates = doc.xpath("//div[contains(concat(' ', normalize-space(@class), ' '), ' idsaPracticeGuidelinePage ')]")
-    if not candidates:
-        candidates = doc.xpath("//*[contains(concat(' ', normalize-space(@class), ' '), ' body-container ')]")
-    if not candidates:
-        candidates = doc.xpath("//div[contains(concat(' ', normalize-space(@class), ' '), ' standardpage-col-left ')]")
-    if not candidates:
-        raise IDSAFetchError("Page markup changed (no IDSA guideline content container)")
-    content = candidates[-1]
-    _remove_noise(content)
-    content_html = lxml_html.tostring(content, encoding="unicode")
-    if not clean_text(content.text_content(), drop_numeric_citations=False):
-        raise IDSAFetchError("Page markup changed (empty IDSA guideline content container)")
-    return content_html
-
-
-def _remove_noise(content: lxml_html.HtmlElement) -> None:
-    noise_xpath = (
-        ".//*[self::script or self::style or self::noscript or self::svg or self::button"
-        " or contains(concat(' ', normalize-space(@class), ' '), ' table-of-contents ')"
-        " or contains(concat(' ', normalize-space(@class), ' '), ' toc ')"
-        " or contains(concat(' ', normalize-space(@class), ' '), ' TableOfContents ')"
-        " or contains(concat(' ', normalize-space(@class), ' '), ' status-section ')"
-        " or contains(concat(' ', normalize-space(@class), ' '), ' view-all-guidance ')"
-        " or contains(concat(' ', normalize-space(@class), ' '), ' share ')"
-        " or contains(concat(' ', normalize-space(@class), ' '), ' addthis ')"
-        " or contains(concat(' ', normalize-space(@class), ' '), ' social ')]"
-    )
-    for element in content.xpath(noise_xpath):
-        _drop_element(element)
-    _remove_table_of_contents(content)
-    for element in content.xpath(".//*[self::a or self::p or self::div or self::span]"):
-        if _BACK_TO_TOP_RE.match(element.text_content()):
-            _drop_element(element)
-
-
-def _remove_table_of_contents(content: lxml_html.HtmlElement) -> None:
-    headings = content.xpath(".//*[self::h1 or self::h2 or self::h3 or self::h4 or self::h5 or self::h6]")
-    for heading in headings:
-        if not _TABLE_OF_CONTENTS_RE.match(heading.text_content()):
-            continue
-        for sibling in list(heading.itersiblings()):
-            if _heading_level(sibling) is not None:
-                break
-            _drop_element(sibling)
-        _drop_element(heading)
-
-
-def _heading_level(element: lxml_html.HtmlElement) -> int | None:
-    tag = element.tag.lower() if isinstance(element.tag, str) else ""
-    if len(tag) == 2 and tag.startswith("h") and tag[1].isdigit():
-        return int(tag[1])
-    return None
-
-
-def _drop_element(element: lxml_html.HtmlElement) -> None:
-    parent = element.getparent()
-    if parent is not None:
-        parent.remove(element)
-
-
-def _links_metadata(content_html: str) -> dict[str, list[str]]:
-    doc = lxml_html.fromstring(content_html)
-    external_links: list[str] = []
-    pdf_links: list[str] = []
-    for link in doc.xpath(".//a[@href]"):
-        url = urljoin(BASE_URL, link.get("href")).split("#")[0].split("?")[0]
-        if urlparse(url).netloc.lower() in {"www.idsociety.org", "idsociety.org"}:
-            continue
-        external_links.append(url)
-        link_text = _WHITESPACE_RE.sub(" ", link.text_content()).strip()
-        if url.lower().endswith(".pdf") or _PDF_TEXT_RE.search(link_text):
-            pdf_links.append(url)
-    return {
-        "external_links": absolute_unique_urls(external_links, base_url=BASE_URL),
-        "pdf_links": absolute_unique_urls(pdf_links, base_url=BASE_URL),
-    }
-
-
-def _section_count(content_html: str) -> int:
-    doc = lxml_html.fromstring(content_html)
-    headings = doc.xpath(".//*[self::h1 or self::h2 or self::h3]")
-    return max(1, len(headings))
-
-
-def _quality_flags(content: str) -> list[str]:
-    flags: list[str] = []
-    if len(content) < _SHORT_CONTENT_CHARS:
-        flags.append("short_content")
-    return flags
-
-
-def _page_url(slug: str) -> str:
-    return f"{BASE_URL}/practice-guideline/{slug}/"
-
-
-__all__ = [
-    "BASE_URL",
-    "DOCUMENT_DELAY_SECONDS",
-    "IDSAFetchError",
-    "IDSAGuidelineListingPage",
-    "IDSAGuidelineRef",
-    "LISTING_URL",
-    "idsa_ref_from_url",
-    "list_practice_guidelines",
-    "scrape_guideline",
-    "scrape_idsa",
-]
diff --git a/datasets/amfv_datasets/scraping/pubmed.py b/datasets/amfv_datasets/scraping/pubmed.py
deleted file mode 100644
index 5a1f411b..00000000
--- a/datasets/amfv_datasets/scraping/pubmed.py
+++ /dev/null
@@ -1,723 +0,0 @@
-"""Scrape PubMed clinical practice guidelines into normalized markdown documents.
-
-PubMed indexes roughly 43k articles tagged with the `Guideline` publication
-type, but most of them expose an abstract only and stay under publisher
-copyright. This module deliberately scrapes the intersection with the PMC Open
-Access subset instead:
-
-    Guideline[pt] AND pubmed pmc open access[filter] AND English[la]
-
-That is about 3,000 guidelines, growing by a handful a week, and it is the only
-slice where the full text is both retrievable and openly licensed.
-
-Licensing is recorded per document rather than claimed for the source, because
-the Open Access Subset is not uniformly Creative Commons licensed. Censused
-over all 2,999 records present on 2026-07-29: CC BY 44.5%, CC BY-NC 22.1%,
-CC BY-NC-ND 18.4%, CC BY-NC-SA 2.1%, CC0 1.3%, and 11.7% carrying publisher
-terms instead. By what that permits, 45.8% is unrestricted for derivative
-works, 24.1% is non-commercial only, 18.4% asserts NoDerivatives, and 11.7%
-has to be read case by case.
-
-Discovery and extraction both go through NCBI's E-utilities rather than the
-rendered pubmed.ncbi.nlm.nih.gov pages. `esearch` paginates by numeric offset and
-reports the result total, `esummary` resolves a whole batch of PMIDs to PMCIDs
-and citation metadata in one request, and `efetch` returns JATS XML carrying the
-article body, section structure and license. All three are documented, versioned
-contracts that do not break when the website is restyled.
-
-Attribution:
-Meditron's guideline scrapers (epfLLM/meditron, gap-replay/guidelines) cover a
-dozen sources but have no PubMed scraper, so this module has no upstream port to
-follow. It follows the conventions of `nice.py` in this package instead.
-Source license: Apache License 2.0.
-"""
-
-from __future__ import annotations
-
-import copy
-import logging
-import os
-import re
-import time
-from collections.abc import Iterable
-from dataclasses import dataclass
-from typing import Any
-from urllib.parse import urlparse
-
-import httpx
-from lxml import etree
-
-from amfv_datasets.scraping.base import (
-    ScrapedDocument,
-    ScrapeError,
-    ScrapeRun,
-    default_client,
-    scrape_listing_documents,
-)
-from amfv_datasets.scraping.html import LinkMode, clean_text, html_to_markdown
-
-BASE_URL = "https://pubmed.ncbi.nlm.nih.gov"
-EUTILS_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"
-PMC_ARTICLE_URL = "https://www.ncbi.nlm.nih.gov/pmc/articles"
-# `Guideline` is a strict superset of `Practice Guideline`: searching for either
-# returns exactly the `Guideline` count. The 352 extra records are clinical, not
-# administrative — the 2025 Korean CPR guidelines and similar — so scoping to
-# `Practice Guideline` would drop 352 records (11%) for nothing.
-#
-# `English[la]` drops 185 records. Most other sources in this package are
-# English-only as a side effect of the URL they start from (CPS is a bilingual
-# site scraped through its `/en/` routes; WHO publishes in six languages and is
-# scraped through its English listing). PubMed's API returns every language, so
-# the filter is explicit here to match. The excluded records are largely French
-# CMAJ translations of English guidelines already in the corpus.
-SEARCH_TERM = "Guideline[pt] AND pubmed pmc open access[filter] AND English[la]"
-PUBMED_DATASET_NAME = "pubmed-webscrape"
-PUBMED_DATASET_DISPLAY_NAME = "PubMed Webscrape"
-NCBI_TOOL = "amfv"
-# NCBI documents 3 requests/second without an API key and 10 with one, so unlike
-# a scraped website this delay is a published allowance rather than a guess. Do
-# not raise it to match the other scrapers "for consistency" — that turns a
-# 20-minute run into a 4-hour one for no benefit.
-DOCUMENT_DELAY_SECONDS = 0.4
-LISTING_PAGE_SIZE = 200
-# Enough of a non-CC license statement to tell "unrestricted re-use" apart from
-# "all rights reserved" without carrying a wall of boilerplate per document.
-_LICENSE_STATEMENT_CHARS = 400
-_FIGURE_CAPTION_CHARS = 300
-# NCBI answers a burst with 429 and is transiently unavailable often enough that
-# a multi-thousand document run needs to ride both out.
-_MAX_RETRIES = 4
-_RETRY_BACKOFF_SECONDS = 1.0
-_RETRYABLE_STATUS = frozenset({429, 500, 502, 503, 504})
-
-logger = logging.getLogger(__name__)
-
-_PUBMED_PATH_RE = re.compile(r"^/(?P<pmid>\d+)/?$")
-_PMC_PATH_RE = re.compile(r"^/pmc/articles/(?P<pmcid>PMC\d+)/?$", re.IGNORECASE)
-_CC_URL_RE = re.compile(r"https?://creativecommons\.org/\S+?(?=[\s\"<)]|$)")
-# Unwrapping `xref` and dropping figures leaves the surrounding spacing behind,
-# as in "(PRISMA flow diagram in Supplementary Fig. S1) ." — close it back up.
-_ORPHAN_SPACE_RE = re.compile(r" +([,.;:)\]])")
-_XLINK_HREF = "{http://www.w3.org/1999/xlink}href"
-_ALI_LICENSE_REF = "{http://www.niso.org/schemas/ali/1.0/}license_ref"
-
-# JATS elements whose entire subtree is dropped. Reference lists and footnote
-# groups are citation apparatus rather than clinical content. Figure payloads
-# cannot be linked from the API response at all (see `_article_figures`), so the
-# `<graphic>` goes but its `<label>` and `<caption>` stay in the text, and the
-# filename is recorded in `metadata.figures` for a later pass to resolve.
-_DROPPED_TAGS = (
-    "ref-list",
-    "fn-group",
-    "supplementary-material",
-    "table-wrap-foot",
-    "graphic",
-    "inline-graphic",
-    "media",
-    "object-id",
-    "alt-text",
-)
-# Elements unwrapped to their text. `xref` is a citation/figure pointer whose
-# rendered text is noise like "[12]", which `html_to_markdown` then strips.
-_UNWRAPPED_TAGS = ("xref", "styled-content", "named-content")
-_RENAMED_TAGS = {
-    "sec": "div",
-    "italic": "em",
-    "bold": "strong",
-    "underline": "u",
-    "monospace": "code",
-    "list-item": "li",
-    "table-wrap": "div",
-    "disp-quote": "blockquote",
-    "break": "br",
-    "label": "strong",
-}
-
-
-class PubMedFetchError(ScrapeError):
-    """Raised when a guideline cannot be sourced from PubMed or PMC."""
-
-
-@dataclass(frozen=True)
-class PubMedArticleRef:
-    """A PubMed guideline reference resolved from the search listing."""
-
-    pmid: str = ""
-    pmcid: str = ""
-    title: str = ""
-    journal: str = ""
-    pub_date: str = ""
-    doi: str = ""
-
-    @property
-    def page_url(self) -> str:
-        """Canonical PubMed record URL, falling back to the PMC article URL."""
-        return f"{BASE_URL}/{self.pmid}/" if self.pmid else self.pmc_url
-
-    @property
-    def pmc_url(self) -> str:
-        """PMC full-text article URL."""
-        return f"{PMC_ARTICLE_URL}/{self.pmcid}/"
-
-    @property
-    def slug(self) -> str:
-        """Stable identifier used for `external_id`."""
-        return self.pmid or self.pmcid
-
-
-@dataclass(frozen=True)
-class PubMedListingPage:
-    """One page of PubMed guideline search results."""
-
-    refs: list[PubMedArticleRef]
-    total: int | None
-
-
-def pubmed_ref_from_url(url: str) -> PubMedArticleRef:
-    """Parse a PubMed or PMC article URL into a guideline reference.
-
-    Only one of the two identifiers is known from a URL; the other is resolved
-    against E-utilities when the article is scraped.
-
-    Args:
-        url: PubMed or PMC article URL to parse.
-    """
-    parsed = urlparse(url.strip())
-    netloc = parsed.netloc.lower()
-    if parsed.scheme not in {"http", "https"}:
-        raise PubMedFetchError(f"Enter an http(s) PubMed or PMC URL; got {url!r}")
-
-    if netloc in {"pubmed.ncbi.nlm.nih.gov", "www.pubmed.ncbi.nlm.nih.gov"}:
-        match = _PUBMED_PATH_RE.match(parsed.path)
-        if match:
-            return PubMedArticleRef(pmid=match.group("pmid"))
-    elif netloc in {"www.ncbi.nlm.nih.gov", "ncbi.nlm.nih.gov", "pmc.ncbi.nlm.nih.gov"}:
-        match = _PMC_PATH_RE.match(parsed.path)
-        if match:
-            return PubMedArticleRef(pmcid=match.group("pmcid").upper())
-
-    raise PubMedFetchError(f"Enter a URL like {BASE_URL}/42476581/ or {PMC_ARTICLE_URL}/PMC13384732/; got {url!r}")
-
-
-def _policy_params() -> dict[str, str]:
-    """Identify the client per NCBI's E-utilities usage policy.
-
-    NCBI asks every request to carry a `tool` name and contact `email`, and
-    raises the rate limit from 3 to 10 requests/second when an `api_key` is
-    supplied. Both optional values come from the environment so that no personal
-    address is committed to the repository.
-    """
-    params = {"tool": NCBI_TOOL}
-    for name, variable in (("email", "NCBI_EMAIL"), ("api_key", "NCBI_API_KEY")):
-        value = os.environ.get(variable)
-        if value:
-            params[name] = value
-    return params
-
-
-def _eutils_get(client: httpx.Client, endpoint: str, params: dict[str, Any]) -> httpx.Response:
-    """Call one E-utilities endpoint, retrying when NCBI asks us to slow down.
-
-    `DOCUMENT_DELAY_SECONDS` paces one document against the next, but a single
-    document still makes up to two calls back to back, and the published 3/s
-    limit is per source address rather than per process. NCBI answers a burst
-    with 429. Without a retry that propagates out of `_scrape_or_skip`, which
-    only swallows `PubMedFetchError`, and kills a 3,000-document run partway
-    through.
-    """
-    url = f"{EUTILS_URL}/{endpoint}"
-    merged = {**params, **_policy_params()}
-    for attempt in range(_MAX_RETRIES):
-        response = client.get(url, params=merged)
-        if response.status_code in _RETRYABLE_STATUS and attempt < _MAX_RETRIES - 1:
-            delay = _RETRY_BACKOFF_SECONDS * 2**attempt
-            logger.warning("NCBI returned %s for %s; retrying in %.1fs", response.status_code, endpoint, delay)
-            time.sleep(delay)
-            continue
-        response.raise_for_status()
-        return response
-    raise PubMedFetchError(f"E-utilities {endpoint} did not succeed after {_MAX_RETRIES} attempts")
-
-
-def _summarize(client: httpx.Client, pmids: list[str]) -> list[PubMedArticleRef]:
-    """Resolve a batch of PMIDs to references carrying PMCIDs and citation data."""
-    payload = _eutils_get(
-        client,
-        "esummary.fcgi",
-        {"db": "pubmed", "id": ",".join(pmids), "retmode": "json"},
-    ).json()
-    result = payload.get("result", {})
-
-    refs: list[PubMedArticleRef] = []
-    for pmid in result.get("uids", []):
-        record = result.get(pmid, {})
-        article_ids = {entry.get("idtype"): entry.get("value") for entry in record.get("articleids", [])}
-        pmcid = article_ids.get("pmc", "")
-        if not pmcid:
-            # The open-access filter should guarantee a PMC copy, so a missing
-            # one means the record changed between search and summary.
-            logger.warning("Skipping PubMed record %s with no PMC identifier", pmid)
-            continue
-        refs.append(
-            PubMedArticleRef(
-                pmid=pmid,
-                pmcid=pmcid,
-                title=clean_text(record.get("title", ""), drop_numeric_citations=False),
-                journal=record.get("fulljournalname", ""),
-                pub_date=record.get("pubdate", ""),
-                doi=article_ids.get("doi", ""),
-            )
-        )
-    return refs
-
-
-def list_pubmed_guidelines(client: httpx.Client, page: int = 1) -> PubMedListingPage:
-    """List one page of open-access PubMed practice guidelines.
-
-    `esearch` paginates by numeric offset, so the page number maps directly onto
-    `retstart` and a page past the end returns an empty list rather than an
-    error. That is what terminates the scrape loop.
-
-    Args:
-        client: HTTP client used to call E-utilities.
-        page: 1-based listing page number (default: 1).
-    """
-    payload = _eutils_get(
-        client,
-        "esearch.fcgi",
-        {
-            "db": "pubmed",
-            "term": SEARCH_TERM,
-            "retstart": (page - 1) * LISTING_PAGE_SIZE,
-            "retmax": LISTING_PAGE_SIZE,
-            "retmode": "json",
-        },
-    ).json()
-    result = payload.get("esearchresult", {})
-    if "ERROR" in result:
-        raise PubMedFetchError(f"PubMed search error: {result['ERROR']}")
-
-    raw_total = str(result.get("count", ""))
-    total = int(raw_total) if raw_total.isdigit() else None
-    pmids = [str(pmid) for pmid in result.get("idlist", [])]
-    if not pmids:
-        return PubMedListingPage(refs=[], total=total)
-    return PubMedListingPage(refs=_summarize(client, pmids), total=total)
-
-
-def _fetch_pmc_article(client: httpx.Client, pmcid: str) -> etree._Element:
-    """Fetch one PMC record and return its `<article>` element."""
-    response = _eutils_get(
-        client,
-        "efetch.fcgi",
-        {"db": "pmc", "id": pmcid.upper().removeprefix("PMC"), "retmode": "xml"},
-    )
-    try:
-        root = etree.fromstring(response.content)
-    except etree.XMLSyntaxError as error:
-        raise PubMedFetchError(f"Malformed PMC XML for {pmcid}: {error}") from error
-
-    article = root.find(".//article")
-    if article is None:
-        raise PubMedFetchError(f"No PMC article returned for {pmcid}")
-    return article
-
-
-def _resolve_ref(client: httpx.Client, ref: PubMedArticleRef) -> PubMedArticleRef:
-    """Fill in a PMCID for a reference that only carries a PMID."""
-    if ref.pmcid or not ref.pmid:
-        return ref
-    resolved = _summarize(client, [ref.pmid])
-    if not resolved:
-        raise PubMedFetchError(f"No PMC full text for PubMed record {ref.pmid}")
-    return resolved[0]
-
-
-def _first_text(node: etree._Element, tag: str) -> str:
-    """Return the normalized text of the first matching descendant."""
-    element = node.find(f".//{tag}")
-    return clean_text("".join(element.itertext()), drop_numeric_citations=False) if element is not None else ""
-
-
-def _article_license(article: etree._Element) -> dict[str, str]:
-    """Extract licensing terms from the article's front matter.
-
-    The PMC Open Access Subset is not uniformly Creative Commons licensed. 88.3%
-    of the guideline slice carries a CC license, found in one of three shapes:
-    an `<ali:license_ref>` element, an `<ext-link>` pointing at
-    creativecommons.org, or a bare URL in the license free text. The name is
-    parsed from the URL path rather than from the `license-type` attribute,
-    which is unusable: the corpus spells it 18 different ways, including
-    `cc-by`, `creativeCommonsBy`, `CC BY-NC-ND`, `open-access` and `openaccess`.
-
-    The remaining 11.7% carry publisher terms that are not equivalent to each
-    other: most are Elsevier's COVID-19 resource centre grant, whose copyright
-    line still reads "All rights reserved"; some are the PMC Open Access
-    Subset's own "unrestricted re-use" statement; and 112 have no `<license>`
-    element at all, only a plain copyright line such as "(c) Springer-Verlag
-    Tokyo 2007".
-    Being in the subset is therefore not by itself a grant to redistribute, so
-    the raw terms are recorded rather than flattened to an empty license.
-    """
-    # The reservation of rights sits in `<copyright-statement>`, a sibling of
-    # `<license>` rather than a child, so reading only the license element loses
-    # the "All rights reserved" half of a COVID-era temporary grant.
-    copyright_statement = _first_text(article, "copyright-statement")
-
-    license_element = article.find(".//license")
-    if license_element is None:
-        return {
-            "license": "",
-            "license_url": "",
-            "license_type": "",
-            "license_statement": "",
-            "copyright_statement": copyright_statement,
-        }
-
-    url = ""
-    license_ref = license_element.find(f".//{_ALI_LICENSE_REF}")
-    if license_ref is not None and license_ref.text:
-        url = license_ref.text.strip()
-    if not url:
-        for element in license_element.iter():
-            href = element.get(_XLINK_HREF) or ""
-            if "creativecommons.org" in href:
-                url = href.strip()
-                break
-    if not url:
-        match = _CC_URL_RE.search("".join(license_element.itertext()))
-        url = match.group(0) if match else ""
-
-    name = ""
-    match = re.search(r"creativecommons\.org/(?:licenses|publicdomain)/([a-z0-9-]+)/", url)
-    if match:
-        token = match.group(1)
-        name = "CC0" if token == "zero" else f"CC {token.upper()}"
-
-    statement = clean_text("".join(license_element.itertext()), drop_numeric_citations=False)
-    return {
-        "license": name,
-        "license_url": url,
-        "license_type": license_element.get("license-type", ""),
-        "license_statement": statement[:_LICENSE_STATEMENT_CHARS],
-        "copyright_statement": copyright_statement,
-    }
-
-
-def _article_figures(article: etree._Element) -> list[dict[str, str]]:
-    """Record each figure's source filename, label and caption.
-
-    The image itself cannot be linked from here. JATS carries only a bare
-    filename (`keag320f1.jpg`), while the served URL inserts a CDN shard and a
-    content hash — `.../pmc/blobs/1ab4/13384732/bd9a2d61ca1a/keag320f1.jpg` —
-    that appear nowhere in the API response. Building a URL from the filename
-    alone yields a 404, which is worse than no link because nothing downstream
-    notices it is broken.
-
-    Other scrapers here keep images as absolute remote references (see the
-    `<img>` handling added in #9 and #10), which works because their sources
-    serve a real path. Recording the filename keeps that door open: a later
-    pass can resolve these against the rendered page or the OA package without
-    re-scraping the corpus.
-
-    Scans the whole article rather than just the body: a graphical abstract is
-    declared in `<front>`, so a body-only scan silently misses it.
-
-    Args:
-        article: The parsed `<article>` element, before tag rewriting.
-    """
-    figures: list[dict[str, str]] = []
-    for figure in article.iter("fig"):
-        graphic = figure.find(".//graphic")
-        href = graphic.get(_XLINK_HREF) if graphic is not None else None
-        if not href:
-            continue
-        figures.append(
-            {
-                "id": figure.get("id", ""),
-                "file": href,
-                "label": _first_text(figure, "label"),
-                "caption": _first_text(figure, "caption")[:_FIGURE_CAPTION_CHARS],
-            }
-        )
-    return figures
-
-
-def _article_supplements(article: etree._Element) -> list[dict[str, str]]:
-    """Record supplementary file names, labels and captions.
-
-    Same situation as `_article_figures`, and measured: across 40 sampled
-    guidelines every supplementary block was a pointer to an external file
-    rather than inline content, totalling 0.18% of body text. The evidence
-    tables a verifier would want live inside those `.docx` and `.tif` payloads,
-    which the API does not return, so the pointer is the only thing worth
-    keeping.
-
-    Args:
-        article: The parsed `<article>` element, before tag rewriting.
-    """
-    supplements: list[dict[str, str]] = []
-    for supplement in article.iter("supplementary-material"):
-        href = supplement.get(_XLINK_HREF)
-        if not href:
-            media = supplement.find(f".//*[@{_XLINK_HREF}]")
-            href = media.get(_XLINK_HREF) if media is not None else None
-        if not href:
-            continue
-        supplements.append(
-            {
-                "id": supplement.get("id", ""),
-                "file": href,
-                "label": _first_text(supplement, "label"),
-                "caption": _first_text(supplement, "caption")[:_FIGURE_CAPTION_CHARS],
-            }
-        )
-    return supplements
-
-
-def _heading_depth(section: etree._Element) -> int:
-    """Return how deeply a `<sec>` is nested inside the article body."""
-    depth = 1
-    parent = section.getparent()
-    while parent is not None:
-        if parent.tag == "sec":
-            depth += 1
-        parent = parent.getparent()
-    return depth
-
-
-def _jats_to_html(source: etree._Element) -> str:
-    """Rewrite a JATS subtree into HTML that `html_to_markdown` understands.
-
-    JATS is close enough to HTML that renaming tags is cheaper and less
-    error-prone than writing a second markdown serializer: `<table-wrap>`
-    already contains genuine XHTML tables, and inline markup maps one-to-one.
-    """
-    root = copy.deepcopy(source)
-    etree.strip_elements(root, *_DROPPED_TAGS, with_tail=False)
-    etree.strip_tags(root, *_UNWRAPPED_TAGS)
-
-    # An `<abstract>` labels itself "Abstract", which would duplicate the heading
-    # this module already writes above it.
-    if root.tag == "abstract":
-        title = root.find("title")
-        if title is not None:
-            root.remove(title)
-
-    # Headings depend on `<sec>` nesting, so resolve them before `sec` is renamed.
-    for section in root.iter("sec"):
-        title = section.find("title")
-        if title is not None:
-            title.tag = f"h{min(_heading_depth(section) + 1, 6)}"
-    for title in root.iter("title"):
-        title.tag = "strong"
-
-    for element in root.iter():
-        if not isinstance(element.tag, str):
-            continue
-        if element.tag == "list":
-            element.tag = "ol" if element.get("list-type") == "order" else "ul"
-        elif element.tag == "ext-link":
-            href = element.get(_XLINK_HREF)
-            element.tag = "a"
-            if href:
-                element.set("href", href)
-        else:
-            element.tag = _RENAMED_TAGS.get(element.tag, element.tag)
-        for name in list(element.attrib):
-            if name.startswith("{") or name in {"list-type", "content-type", "position"}:
-                del element.attrib[name]
-
-    return etree.tostring(root, encoding="unicode", method="html")
-
-
-def _jats_to_markdown(source: etree._Element, *, link_mode: LinkMode) -> str:
-    """Convert a JATS subtree to markdown and close up gaps left by dropped nodes.
-
-    The spacing pass has to run last: `html_to_markdown` is what removes the
-    "[12]" text of an unwrapped `xref`, so the orphaned space only appears once
-    the markdown exists.
-    """
-    markdown = html_to_markdown(_jats_to_html(source), link_mode=link_mode)
-    return _ORPHAN_SPACE_RE.sub(r"\1", markdown)
-
-
-def build_pubmed_article_text(
-    client: httpx.Client,
-    ref: PubMedArticleRef,
-    *,
-    link_mode: LinkMode = LinkMode.KEEP,
-) -> tuple[str, int, str, dict[str, Any]]:
-    """Scrape one guideline's full text into markdown, section count, title and metadata.
-
-    Args:
-        client: HTTP client used to call E-utilities.
-        ref: PubMed guideline reference to scrape.
-        link_mode: Whether links are kept as markdown links or stripped to their
-            visible text (default: LinkMode.KEEP).
-    """
-    ref = _resolve_ref(client, ref)
-    article = _fetch_pmc_article(client, ref.pmcid)
-
-    body = article.find(".//body")
-    if body is None:
-        # Roughly 2% of the subset is deposited as metadata only. That is a
-        # property of the record, not a fetch failure.
-        raise PubMedFetchError(f"No full text body in PMC record {ref.pmcid}")
-
-    sections: list[str] = []
-    abstract = article.find(".//abstract")
-    if abstract is not None:
-        abstract_markdown = _jats_to_markdown(abstract, link_mode=link_mode)
-        if abstract_markdown:
-            sections.append(f"## Abstract\n\n{abstract_markdown}")
-
-    body_markdown = _jats_to_markdown(body, link_mode=link_mode)
-    if body_markdown:
-        sections.append(body_markdown)
-
-    content = "\n\n".join(sections).strip()
-    if not content:
-        raise PubMedFetchError(f"No readable content in PMC record {ref.pmcid}")
-
-    title = ref.title or _first_text(article, "article-title") or ref.slug
-    section_count = max(len(body.findall("sec")), 1) + (1 if abstract is not None else 0)
-    metadata = {
-        "pmid": ref.pmid or _pmid_from_article(article),
-        "pmcid": ref.pmcid,
-        "doi": ref.doi or _article_id(article, "doi"),
-        "journal": ref.journal or _first_text(article, "journal-title"),
-        "pub_date": ref.pub_date,
-        "article_type": article.get("article-type", ""),
-        **_article_license(article),
-        "figures": _article_figures(article),
-        "supplements": _article_supplements(article),
-        "content_length_chars": len(content),
-    }
-    return content, section_count, title, metadata
-
-
-def _article_id(article: etree._Element, id_type: str) -> str:
-    """Return an `<article-id>` value of the requested type."""
-    for element in article.iter("article-id"):
-        if element.get("pub-id-type") == id_type:
-            return "".join(element.itertext()).strip()
-    return ""
-
-
-def _pmid_from_article(article: etree._Element) -> str:
-    return _article_id(article, "pmid")
-
-
-def scrape_pubmed_article(
-    client: httpx.Client,
-    ref: PubMedArticleRef,
-    *,
-    link_mode: LinkMode = LinkMode.KEEP,
-) -> ScrapedDocument:
-    """Scrape a PubMed guideline into a normalized document.
-
-    Args:
-        client: HTTP client used to call E-utilities.
-        ref: PubMed guideline reference to scrape.
-        link_mode: Whether links are kept as markdown links or stripped to their
-            visible text (default: LinkMode.KEEP).
-    """
-    content, section_count, title, metadata = build_pubmed_article_text(client, ref, link_mode=link_mode)
-    # Prefer the PMID from the record itself so one article gets one identifier
-    # whether it was reached from the listing or from a PMC URL, which carries
-    # no PMID of its own.
-    slug = metadata["pmid"] or ref.slug or metadata["pmcid"]
-    return ScrapedDocument(
-        source="pubmed",
-        external_id=f"pubmed-{slug}",
-        title=title,
-        url=ref.page_url,
-        content=content,
-        section_count=section_count,
-        metadata=metadata,
-    )
-
-
-def _scrape_or_skip(
-    client: httpx.Client,
-    ref: PubMedArticleRef,
-    *,
-    link_mode: LinkMode,
-) -> ScrapedDocument | None:
-    """Scrape one guideline, skipping records PMC holds without full text.
-
-    About 2% of the open-access subset is deposited as metadata only, so a run
-    over the whole corpus should not die when it reaches one.
-
-    Args:
-        client: HTTP client used to call E-utilities.
-        ref: PubMed guideline reference to scrape.
-        link_mode: Whether links are kept as markdown links or stripped to their
-            visible text.
-    """
-    try:
-        return scrape_pubmed_article(client, ref, link_mode=link_mode)
-    except PubMedFetchError:
-        logger.warning("Skipping PubMed guideline %r", ref.slug, exc_info=True)
-        return None
-
-
-def scrape_pubmed(
-    *,
-    documents: int | None,
-    link_mode: LinkMode = LinkMode.KEEP,
-    url: str | None = None,
-) -> ScrapeRun:
-    """Scrape PubMed guidelines from a URL or the open-access search listing.
-
-    Args:
-        documents: Number of documents to scrape. Ignored when `url` is set.
-            When unset, listing pages are fetched until a page returns no items
-            (default: None).
-        link_mode: Whether links are kept as markdown links or stripped to their
-            visible text (default: LinkMode.KEEP).
-        url: PubMed or PMC article URL to scrape as a single document
-            (default: None).
-    """
-    if url is not None:
-
-        def scrape_url() -> Iterable[ScrapedDocument]:
-            with default_client() as client:
-                yield scrape_pubmed_article(client, pubmed_ref_from_url(url), link_mode=link_mode)
-
-        return ScrapeRun(documents=scrape_url(), total=1)
-
-    with default_client() as client:
-        first_page = list_pubmed_guidelines(client, page=1)
-    total = first_page.total if documents is None or first_page.total is None else min(documents, first_page.total)
-    return ScrapeRun(
-        total=total,
-        documents=scrape_listing_documents(
-            documents=documents,
-            client_factory=default_client,
-            first_page_items=first_page.refs,
-            list_page=lambda client, page: list_pubmed_guidelines(client, page).refs,
-            scrape_item=lambda client, ref: _scrape_or_skip(client, ref, link_mode=link_mode),
-            document_delay_seconds=DOCUMENT_DELAY_SECONDS,
-        ),
-    )
-
-
-__all__ = [
-    "BASE_URL",
-    "DOCUMENT_DELAY_SECONDS",
-    "EUTILS_URL",
-    "PMC_ARTICLE_URL",
-    "PUBMED_DATASET_DISPLAY_NAME",
-    "PUBMED_DATASET_NAME",
-    "SEARCH_TERM",
-    "PubMedArticleRef",
-    "PubMedFetchError",
-    "PubMedListingPage",
-    "build_pubmed_article_text",
-    "list_pubmed_guidelines",
-    "pubmed_ref_from_url",
-    "scrape_pubmed",
-    "scrape_pubmed_article",
-]
diff --git a/datasets/amfv_datasets/scraping/rch.py b/datasets/amfv_datasets/scraping/rch.py
deleted file mode 100644
index db1e1302..00000000
--- a/datasets/amfv_datasets/scraping/rch.py
+++ /dev/null
@@ -1,398 +0,0 @@
-"""Scrape RCH clinical practice guidelines into normalized markdown documents.
-
-The Royal Children's Hospital Melbourne (RCH) publishes its clinical practice
-guidelines as structured HTML pages. This module discovers guideline pages from
-the A-Z index and extracts only the primary guideline and reference widgets.
-"""
-
-from __future__ import annotations
-
-import logging
-import re
-from collections.abc import Collection, Iterable
-from dataclasses import dataclass, replace
-from urllib.parse import unquote, urljoin, urlparse
-
-import httpx
-from lxml import html as lxml_html
-
-from amfv_datasets.scraping.base import (
-    ScrapedDocument,
-    ScrapeError,
-    ScrapeRun,
-    default_client,
-    scrape_listing_documents,
-)
-from amfv_datasets.scraping.html import LinkMode, clean_text, document_title, html_to_markdown
-
-BASE_URL = "https://www.rch.org.au"
-GUIDELINE_INDEX_URL = f"{BASE_URL}/clinicalguide/guideline_index/"
-RCH_DATASET_NAME = "rch-webscrape"
-RCH_DATASET_DISPLAY_NAME = "RCH Clinical Practice Guidelines Webscrape"
-DOCUMENT_DELAY_SECONDS = 10.0
-_LOGGER = logging.getLogger(__name__)
-
-_NON_GUIDELINE_SLUGS = {
-    "CPG_Committee_Calendar",
-    "Fractures",
-    "Immigrant_health_resources",
-}
-
-_GUIDELINE_PATH_RE = re.compile(
-    r"^/clinicalguide/guideline_index/(?P<slug>.+?)/?$",
-    re.IGNORECASE,
-)
-_DIRECT_GUIDELINE_PATHS = {
-    "/persistent_nasal_discharge_rhinosinusitis/.aspx": (
-        "Persistent_nasal_discharge_rhinosinusitis",
-        "/Persistent_nasal_discharge_rhinosinusitis/.aspx",
-    ),
-}
-_SEE_ALIAS_SUFFIX_RE = re.compile(r"\s+\(see\b.*\)$", re.IGNORECASE)
-_LAST_UPDATED_RE = re.compile(r"\bLast updated\s+([^\n]+)", re.IGNORECASE)
-_EMPTY_MARKDOWN_HEADING_RE = re.compile(r"^#{1,6}\s*$", re.MULTILINE)
-_EMPTY_EMPHASIS_RE = re.compile(r"(?<!\*)\*{4}(?!\*)")
-
-
-class RchFetchError(ScrapeError):
-    """Raised when an RCH guideline cannot be discovered or parsed."""
-
-
-@dataclass(frozen=True)
-class RchGuidelineRef:
-    """A clinical practice guideline discovered from the RCH A-Z index."""
-
-    slug: str
-    title: str
-    page_url: str
-    aliases: tuple[str, ...] = ()
-
-
-def guideline_ref_from_url(url: str, *, title: str | None = None) -> RchGuidelineRef:
-    """Parse an RCH guideline URL into a canonical guideline reference.
-
-    Args:
-        url: RCH clinical guideline URL to parse.
-        title: Optional title discovered from the A-Z index (default: None).
-    """
-    parsed = urlparse(url.strip())
-    if parsed.scheme not in {"http", "https"} or parsed.netloc.lower() not in {
-        "www.rch.org.au",
-        "rch.org.au",
-    }:
-        raise RchFetchError(f"Enter an RCH clinical guideline URL from rch.org.au; got {url!r}")
-
-    path = unquote(parsed.path)
-    match = _GUIDELINE_PATH_RE.match(path)
-    if not match:
-        direct_guideline = _DIRECT_GUIDELINE_PATHS.get(path.rstrip("/").casefold())
-        if direct_guideline is not None:
-            slug, canonical_path = direct_guideline
-            return RchGuidelineRef(
-                slug=slug,
-                title=title or slug.replace("_", " "),
-                page_url=f"{BASE_URL}{canonical_path}",
-            )
-        raise RchFetchError(
-            f"Enter a URL like https://www.rch.org.au/clinicalguide/guideline_index/Acute_asthma/; got {url!r}"
-        )
-
-    slug = match.group("slug")
-    if slug.lower().endswith((".pdf", ".doc", ".docx")):
-        raise RchFetchError(f"Enter an RCH HTML clinical guideline URL, not a downloadable file; got {url!r}")
-    trailing_slash = "" if slug.lower().endswith(".aspx") else "/"
-    page_url = f"{BASE_URL}/clinicalguide/guideline_index/{slug}{trailing_slash}"
-    return RchGuidelineRef(slug=slug, title=title or slug.replace("_", " "), page_url=page_url)
-
-
-def list_guidelines(client: httpx.Client) -> list[RchGuidelineRef]:
-    """Return unique guideline references from the RCH A-Z index.
-
-    Args:
-        client: HTTP client used to fetch the guideline index.
-    """
-    response = client.get(GUIDELINE_INDEX_URL)
-    response.raise_for_status()
-    doc = lxml_html.fromstring(response.text)
-    anchors = doc.xpath(
-        "//div[@id='tabnav-letter-blocks' or "
-        "contains(concat(' ', normalize-space(@class), ' '), ' tabnav-letter-blocks ')]//a[@href]"
-    )
-
-    refs_by_url: dict[str, RchGuidelineRef] = {}
-    for anchor in anchors:
-        href = anchor.get("href")
-        title = clean_text(anchor.text_content(), drop_numeric_citations=False)
-        if not href or not title:
-            continue
-        title = _SEE_ALIAS_SUFFIX_RE.sub("", title)
-        try:
-            ref = guideline_ref_from_url(urljoin(BASE_URL, href), title=title)
-        except RchFetchError:
-            continue
-        if ref.slug in _NON_GUIDELINE_SLUGS:
-            continue
-        url_key = ref.page_url.casefold()
-        if existing := refs_by_url.get(url_key):
-            if title != existing.title and title not in existing.aliases:
-                refs_by_url[url_key] = replace(existing, aliases=(*existing.aliases, title))
-            continue
-        refs_by_url[url_key] = ref
-    refs = list(refs_by_url.values())
-    if not refs:
-        raise RchFetchError("Could not find clinical guideline links in the RCH A-Z index")
-    return refs
-
-
-def _content_widgets(html_text: str) -> list[lxml_html.HtmlElement]:
-    doc = lxml_html.fromstring(html_text)
-    widgets = doc.xpath(
-        "//div[@id='rch-primary' or "
-        "contains(concat(' ', normalize-space(@class), ' '), ' rch-primary ')]"
-        "//div[contains(concat(' ', normalize-space(@class), ' '), ' widgetBody ')]"
-    )
-    return [widget for widget in widgets if _is_guideline_widget(widget)]
-
-
-def _is_guideline_widget(widget: lxml_html.HtmlElement) -> bool:
-    if widget.xpath(".//*[self::h2 or self::h3 or self::h4 or self::h5 or self::h6]"):
-        return True
-    text_length = len(clean_text(widget.text_content(), drop_numeric_citations=False))
-    return text_length >= 500 or bool(widget.xpath(".//table")) and text_length >= 200
-
-
-def _normalize_widget(widget: lxml_html.HtmlElement) -> None:
-    for element in widget.xpath(".//script | .//style"):
-        element.drop_tree()
-
-    for nested in widget.xpath(".//strong//strong | .//em//em"):
-        nested.drop_tag()
-
-    for image in widget.xpath(".//img[not(@src) or not(normalize-space(@src))]"):
-        image.drop_tree()
-
-    for element, attribute in (
-        *((link, "href") for link in widget.xpath(".//a[@href]")),
-        *((image, "src") for image in widget.xpath(".//img[@src]")),
-    ):
-        value = element.get(attribute)
-        parsed = urlparse(value)
-        if parsed.netloc.lower() in {"rch.org.au", "www.rch.org.au", "webedit.rch.org.au"}:
-            element.set(attribute, parsed._replace(scheme="https", netloc="www.rch.org.au").geturl())
-
-    for emphasis in widget.xpath(".//strong | .//em | .//b | .//i"):
-        if not clean_text(emphasis.text_content(), drop_numeric_citations=False) and not emphasis.xpath(".//img"):
-            emphasis.drop_tree()
-
-    nested_lists = widget.xpath(".//ul/ul | .//ul/ol | .//ol/ul | .//ol/ol")
-    for nested_list in nested_lists:
-        previous = nested_list.getprevious()
-        if previous is not None and previous.tag.lower() == "li":
-            previous.append(nested_list)
-
-    for heading in widget.xpath(".//*[self::h1 or self::h2 or self::h3 or self::h4 or self::h5 or self::h6]"):
-        for emphasis in heading.xpath(".//strong | .//em | .//b | .//i"):
-            emphasis.drop_tag()
-        if clean_text(heading.text_content(), drop_numeric_citations=False):
-            continue
-        if heading.xpath(".//img"):
-            heading.drop_tag()
-        else:
-            heading.drop_tree()
-
-
-def build_guideline_text(
-    html_text: str,
-    *,
-    link_mode: LinkMode = LinkMode.KEEP,
-    base_url: str = BASE_URL,
-) -> tuple[str, int]:
-    """Extract an RCH guideline page into markdown and a section count.
-
-    Args:
-        html_text: RCH guideline page HTML.
-        link_mode: Whether links are kept as markdown links or stripped to their
-            visible text (default: LinkMode.KEEP).
-        base_url: Document URL used to resolve relative links and fragments
-            (default: BASE_URL).
-    """
-    widgets = _content_widgets(html_text)
-    if not widgets:
-        raise RchFetchError("No readable clinical guideline content found on the RCH page")
-
-    sections: list[str] = []
-    for widget in widgets:
-        _normalize_widget(widget)
-        markdown = html_to_markdown(
-            lxml_html.tostring(widget, encoding="unicode"),
-            link_mode=link_mode,
-            base_url=base_url,
-        )
-        markdown = _EMPTY_MARKDOWN_HEADING_RE.sub("", markdown)
-        markdown = _EMPTY_EMPHASIS_RE.sub("", markdown).strip()
-        if markdown:
-            sections.append(markdown)
-    if not sections:
-        raise RchFetchError("No readable clinical guideline content found on the RCH page")
-    return "\n\n".join(sections), len(sections)
-
-
-def scrape_guideline(
-    client: httpx.Client,
-    ref: RchGuidelineRef,
-    *,
-    link_mode: LinkMode = LinkMode.KEEP,
-) -> ScrapedDocument:
-    """Scrape one RCH guideline into a normalized document.
-
-    Args:
-        client: HTTP client used to fetch the guideline page.
-        ref: RCH guideline reference to scrape.
-        link_mode: Whether links are kept as markdown links or stripped to their
-            visible text (default: LinkMode.KEEP).
-    """
-    response = client.get(ref.page_url)
-    response.raise_for_status()
-    content_url = ref.page_url
-    metadata = {"slug": ref.slug}
-    if ref.aliases:
-        metadata["aliases"] = list(ref.aliases)
-    try:
-        content, section_count = build_guideline_text(
-            response.text,
-            link_mode=link_mode,
-            base_url=content_url,
-        )
-    except RchFetchError:
-        content_url = _linked_content_url(response.text, source_url=ref.page_url)
-        if content_url is None:
-            raise
-        response = client.get(content_url)
-        response.raise_for_status()
-        content, section_count = build_guideline_text(
-            response.text,
-            link_mode=link_mode,
-            base_url=content_url,
-        )
-        metadata["index_url"] = ref.page_url
-    title = document_title(response.text, fallback=ref.title)
-    last_updated_match = _LAST_UPDATED_RE.search(content)
-    if last_updated_match:
-        metadata["last_updated"] = last_updated_match.group(1).strip()
-    return ScrapedDocument(
-        source="rch",
-        external_id=f"rch-{_external_id_slug(ref.slug)}",
-        title=title,
-        url=content_url,
-        content=content,
-        section_count=section_count,
-        metadata=metadata,
-    )
-
-
-def _linked_content_url(html_text: str, *, source_url: str) -> str | None:
-    doc = lxml_html.fromstring(html_text)
-    hrefs = doc.xpath(
-        "//div[@id='rch-primary']//div["
-        "contains(concat(' ', normalize-space(@class), ' '), ' widgetBody ')"
-        "]//a[@href]/@href"
-    )
-    candidates: list[str] = []
-    for href in hrefs:
-        target = urljoin(source_url, href)
-        parsed = urlparse(target)
-        if parsed.scheme not in {"http", "https"} or parsed.netloc.lower() not in {
-            "rch.org.au",
-            "www.rch.org.au",
-        }:
-            continue
-        normalized = target.split("#", maxsplit=1)[0]
-        if normalized != source_url and normalized not in candidates:
-            candidates.append(normalized)
-    return candidates[0] if len(candidates) == 1 else None
-
-
-def _external_id_slug(slug: str) -> str:
-    return re.sub(r"[^a-z0-9]+", "-", slug.lower()).strip("-")
-
-
-def scrape_rch(
-    *,
-    documents: int | None,
-    link_mode: LinkMode = LinkMode.KEEP,
-    url: str | None = None,
-    skip_urls: Collection[str] = (),
-) -> ScrapeRun:
-    """Scrape RCH guidelines from a URL or the A-Z index.
-
-    Args:
-        documents: Number of documents to scrape. Ignored when `url` is set.
-            When unset, every guideline in the A-Z index is scraped (default:
-            None).
-        link_mode: Whether links are kept as markdown links or stripped to their
-            visible text (default: LinkMode.KEEP).
-        url: RCH guideline URL to scrape as a single document (default: None).
-        skip_urls: Canonical document URLs already scraped by a previous run
-            (default: ()).
-    """
-    if url is not None:
-
-        def scrape_url() -> Iterable[ScrapedDocument]:
-            with default_client() as client:
-                yield scrape_guideline(client, guideline_ref_from_url(url), link_mode=link_mode)
-
-        return ScrapeRun(documents=scrape_url(), total=1)
-
-    with default_client() as client:
-        refs = list_guidelines(client)
-    refs = [ref for ref in refs if ref.page_url not in skip_urls]
-    total = len(refs) if documents is None else min(documents, len(refs))
-    return ScrapeRun(
-        total=total,
-        documents=scrape_listing_documents(
-            documents=documents,
-            client_factory=default_client,
-            first_page_items=refs,
-            list_page=lambda _client, _page: (),
-            scrape_item=lambda client, ref: _scrape_or_skip_guideline(client, ref, link_mode=link_mode),
-            document_delay_seconds=DOCUMENT_DELAY_SECONDS,
-        ),
-    )
-
-
-def _scrape_or_skip_guideline(
-    client: httpx.Client,
-    ref: RchGuidelineRef,
-    *,
-    link_mode: LinkMode,
-) -> ScrapedDocument | None:
-    try:
-        return scrape_guideline(client, ref, link_mode=link_mode)
-    except RchFetchError as exc:
-        _LOGGER.warning("Skipping unsupported RCH index entry %s: %s", ref.page_url, exc)
-        return None
-    except httpx.HTTPStatusError as exc:
-        if exc.response.status_code not in {404, 410}:
-            raise
-        _LOGGER.warning(
-            "Skipping stale RCH index entry %s: HTTP %s",
-            ref.page_url,
-            exc.response.status_code,
-        )
-        return None
-
-
-__all__ = [
-    "BASE_URL",
-    "DOCUMENT_DELAY_SECONDS",
-    "GUIDELINE_INDEX_URL",
-    "RCH_DATASET_DISPLAY_NAME",
-    "RCH_DATASET_NAME",
-    "RchFetchError",
-    "RchGuidelineRef",
-    "build_guideline_text",
-    "guideline_ref_from_url",
-    "list_guidelines",
-    "scrape_guideline",
-    "scrape_rch",
-]
diff --git a/datasets/amfv_datasets/scraping/who.py b/datasets/amfv_datasets/scraping/who.py
deleted file mode 100644
index 757232d6..00000000
--- a/datasets/amfv_datasets/scraping/who.py
+++ /dev/null
@@ -1,374 +0,0 @@
-"""Scrape WHO guideline publications into normalized markdown documents.
-
-WHO (World Health Organization) publishes guidelines primarily as PDFs hosted on
-iris.who.int. The publication landing pages expose a short HTML Overview section
-plus bibliographic metadata. For Milestone 1 we scrape that Overview text rather
-than PDFs, because PDF extraction loses reading order and interleaves
-headers/footers. Full-text PDF extraction is intentionally deferred.
-
-Discovery uses WHO's Sitefinity OData publications hub API, filtered to the
-Guidelines publishing office. Extraction fetches each publication HTML page and
-converts the Overview block to markdown.
-
-Attribution:
-The publishing-office filter UUID and PDF-first approach in Meditron's WHO
-scraper (epfLLM/meditron, gap-replay/guidelines/scrapers/scrapers.py) informed
-discovery scope. Source license: Apache License 2.0.
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import re
-from collections.abc import Iterable
-from dataclasses import dataclass
-from typing import Any
-from urllib.parse import urlparse
-
-import httpx
-from lxml import html as lxml_html
-
-from amfv_datasets.scraping.base import (
-    ScrapedDocument,
-    ScrapeError,
-    ScrapeRun,
-    default_client,
-    scrape_listing_documents,
-)
-from amfv_datasets.scraping.html import LinkMode, clean_text, document_title, html_to_markdown
-
-BASE_URL = "https://www.who.int"
-GUIDELINES_LISTING_URL = f"{BASE_URL}/publications/who-guidelines"
-GUIDELINES_PUBLISHING_OFFICE = "c09761c0-ab8e-4cfa-9744-99509c4d306b"
-SF_SITE = "15210d59-ad60-47ff-a542-7ed76645f0c7"
-PUBLICATIONS_API_PATH = "/api/hubs/publications"
-LISTING_PAGE_SIZE = 25
-DOCUMENT_DELAY_SECONDS = 5.0
-WHO_LICENSE = "CC BY-NC-SA 3.0 IGO"
-WHO_ATTRIBUTION = "© World Health Organization. Licensed under CC BY-NC-SA 3.0 IGO."
-CONTENT_SCOPE = "overview"
-
-logger = logging.getLogger(__name__)
-
-_PUBLICATION_PATH_RE = re.compile(
-    r"^/publications/i/item/(?P<publication_id>[^/?#]+)(?:/|$)",
-    re.IGNORECASE,
-)
-_ISBN_RE = re.compile(r"ISBN:\s*([\d\-]+)", re.IGNORECASE)
-
-
-class WhoFetchError(ScrapeError):
-    """Raised when a WHO publication cannot be fetched or parsed."""
-
-
-@dataclass(frozen=True)
-class WhoPublicationRef:
-    """A WHO guideline publication reference from the listing."""
-
-    publication_id: str
-    title: str
-    page_url: str
-    publication_date: str | None = None
-    tag: str | None = None
-    download_url: str | None = None
-
-
-@dataclass(frozen=True)
-class WhoListingPage:
-    """A page of WHO guideline publication references."""
-
-    refs: list[WhoPublicationRef]
-    total: int | None
-
-
-def publication_ref_from_url(url: str) -> WhoPublicationRef:
-    """Parse a WHO publication URL into a canonical publication reference.
-
-    Args:
-        url: WHO publication URL to parse.
-    """
-    parsed = urlparse(url.strip())
-    if parsed.scheme not in {"http", "https"} or parsed.netloc.lower() not in {
-        "www.who.int",
-        "who.int",
-    }:
-        raise WhoFetchError(f"Enter a WHO publication URL from who.int; got {url!r}")
-
-    match = _PUBLICATION_PATH_RE.match(parsed.path.rstrip("/") + "/")
-    if not match:
-        raise WhoFetchError(f"Enter a URL like https://www.who.int/publications/i/item/9789240121805; got {url!r}")
-
-    publication_id = match.group("publication_id")
-    return WhoPublicationRef(
-        publication_id=publication_id,
-        title=publication_id,
-        page_url=_page_url(publication_id=publication_id),
-    )
-
-
-def _page_url(*, publication_id: str) -> str:
-    return f"{BASE_URL}/publications/i/item/{publication_id}"
-
-
-def _publications_api_params(*, page: int) -> dict[str, str]:
-    skip = (page - 1) * LISTING_PAGE_SIZE
-    return {
-        "sf_site": SF_SITE,
-        "sf_provider": "OpenAccessProvider",
-        "sf_culture": "en",
-        "$orderby": "PublicationDateAndTime desc",
-        "$select": "Title,ItemDefaultUrl,FormatedDate,Tag,DownloadUrl",
-        "$filter": f"publishingoffices/any(s:s eq {GUIDELINES_PUBLISHING_OFFICE})",
-        "$top": str(LISTING_PAGE_SIZE),
-        "$skip": str(skip),
-        "$count": "true",
-    }
-
-
-def _publication_id_from_item_url(item_default_url: str) -> str:
-    return item_default_url.strip("/").split("/")[-1]
-
-
-def _parse_api_listing(payload: dict[str, Any]) -> WhoListingPage:
-    """Parse a WHO publications OData response into listing refs."""
-    try:
-        items = payload["value"]
-    except KeyError as exc:
-        raise WhoFetchError("Could not parse WHO publications API JSON") from exc
-
-    refs: list[WhoPublicationRef] = []
-    for item in items:
-        item_default_url = (item.get("ItemDefaultUrl") or "").strip()
-        title = (item.get("Title") or "").strip()
-        if not item_default_url or not title:
-            continue
-        publication_id = _publication_id_from_item_url(item_default_url)
-        refs.append(
-            WhoPublicationRef(
-                publication_id=publication_id,
-                title=title,
-                page_url=_page_url(publication_id=publication_id),
-                publication_date=(item.get("FormatedDate") or None),
-                tag=(item.get("Tag") or None),
-                download_url=(item.get("DownloadUrl") or None),
-            )
-        )
-
-    total = payload.get("@odata.count")
-    return WhoListingPage(refs=refs, total=total if isinstance(total, int) else None)
-
-
-def _parse_html_listing(html_text: str) -> WhoListingPage:
-    """Parse server-rendered WHO guideline cards from the listing page."""
-    doc = lxml_html.fromstring(html_text)
-    refs: list[WhoPublicationRef] = []
-    seen: set[str] = set()
-    for link in doc.xpath("//a[contains(@href,'/publications/i/item/')]"):
-        href = link.get("href", "").split("?")[0]
-        match = _PUBLICATION_PATH_RE.match(href.rstrip("/") + "/")
-        if not match or href in seen:
-            continue
-        seen.add(href)
-        publication_id = match.group("publication_id")
-        title = clean_text(link.xpath("string(.)")) or publication_id
-        refs.append(
-            WhoPublicationRef(
-                publication_id=publication_id,
-                title=title,
-                page_url=_page_url(publication_id=publication_id),
-            )
-        )
-    return WhoListingPage(refs=refs, total=None)
-
-
-def list_publications(client: httpx.Client, page: int = 1) -> WhoListingPage:
-    """Return one page of WHO guideline publication refs.
-
-    Args:
-        client: HTTP client used to fetch the publications API.
-        page: Listing page number (default: 1).
-    """
-    response = client.get(f"{BASE_URL}{PUBLICATIONS_API_PATH}", params=_publications_api_params(page=page))
-    if response.status_code >= 400:
-        logger.warning("WHO publications API returned %s; falling back to HTML listing", response.status_code)
-        listing = client.get(GUIDELINES_LISTING_URL)
-        listing.raise_for_status()
-        return _parse_html_listing(listing.text)
-
-    try:
-        payload = response.json()
-    except json.JSONDecodeError as exc:
-        raise WhoFetchError("Could not decode WHO publications API JSON") from exc
-    return _parse_api_listing(payload)
-
-
-def _overview_html(section: lxml_html.HtmlElement) -> str:
-    """Return HTML for the Overview block within a publication section."""
-    overview_headings = section.xpath(".//h3[normalize-space()='Overview']")
-    if not overview_headings:
-        return lxml_html.tostring(section, encoding="unicode")
-
-    heading = overview_headings[0]
-    parts = [lxml_html.tostring(heading, encoding="unicode")]
-    sibling = heading.getnext()
-    while sibling is not None and sibling.tag.lower() not in {"h2", "h3"}:
-        parts.append(lxml_html.tostring(sibling, encoding="unicode"))
-        sibling = sibling.getnext()
-    return "".join(parts)
-
-
-def _page_isbn(html_text: str) -> str | None:
-    doc = lxml_html.fromstring(html_text)
-    for node in doc.xpath("//*[contains(normalize-space(.), 'ISBN')]"):
-        match = _ISBN_RE.search(node.text_content())
-        if match:
-            return match.group(1)
-    return None
-
-
-def _page_publication_date(doc: lxml_html.HtmlElement) -> str | None:
-    values = doc.xpath("//*[contains(@class,'dynamic-content__date')]/text()")
-    return clean_text(values[0]) if values else None
-
-
-def _page_tag(doc: lxml_html.HtmlElement) -> str | None:
-    values = doc.xpath("//*[contains(@class,'dynamic-content__tag')]/text()")
-    text = clean_text(" ".join(values))
-    return text.lstrip("| ").strip() if text else None
-
-
-def _page_download_url(doc: lxml_html.HtmlElement) -> str | None:
-    for href in doc.xpath("//a[contains(@href,'iris.who.int')]/@href"):
-        if "bitstreams" in href:
-            return href.split("?")[0]
-    return None
-
-
-def build_publication_text(
-    client: httpx.Client,
-    ref: WhoPublicationRef,
-    *,
-    link_mode: LinkMode = LinkMode.KEEP,
-) -> tuple[str, int, str, dict[str, Any]]:
-    """Scrape a publication Overview into markdown text and bibliographic metadata.
-
-    Args:
-        client: HTTP client used to fetch the publication page.
-        ref: WHO publication reference to scrape.
-        link_mode: Whether links are kept as markdown links or stripped to their
-            visible text (default: LinkMode.KEEP).
-    """
-    response = client.get(ref.page_url)
-    response.raise_for_status()
-    html_text = response.text
-    doc = lxml_html.fromstring(html_text)
-
-    sections = doc.xpath("//section[contains(@class,'dynamic-content__section')]")
-    if not sections:
-        raise WhoFetchError(f"No publication content section for '{ref.publication_id}'")
-
-    title = ref.title if ref.title != ref.publication_id else document_title(html_text, fallback=ref.publication_id)
-    overview_html = _overview_html(sections[0])
-    markdown = html_to_markdown(overview_html, link_mode=link_mode, base_url=BASE_URL).strip()
-    if not markdown:
-        raise WhoFetchError(f"No readable Overview content for '{ref.publication_id}'")
-
-    extra_metadata: dict[str, Any] = {
-        "publication_date": ref.publication_date or _page_publication_date(doc),
-        "tag": ref.tag or _page_tag(doc),
-        "isbn": _page_isbn(html_text),
-        "download_url": ref.download_url or _page_download_url(doc),
-        "content_scope": CONTENT_SCOPE,
-        "license": WHO_LICENSE,
-        "attribution": WHO_ATTRIBUTION,
-        "listing_category": "who-guidelines",
-    }
-    return markdown, 1, title, extra_metadata
-
-
-def scrape_publication(
-    client: httpx.Client,
-    ref: WhoPublicationRef,
-    *,
-    link_mode: LinkMode = LinkMode.KEEP,
-) -> ScrapedDocument:
-    """Scrape a WHO publication into a normalized document.
-
-    Args:
-        client: HTTP client used to fetch the publication page.
-        ref: WHO publication reference to scrape.
-        link_mode: Whether links are kept as markdown links or stripped to their
-            visible text (default: LinkMode.KEEP).
-    """
-    content, section_count, title, metadata = build_publication_text(client, ref, link_mode=link_mode)
-    return ScrapedDocument(
-        source="who",
-        external_id=f"who-{ref.publication_id}",
-        title=title,
-        url=ref.page_url,
-        content=content,
-        section_count=section_count,
-        metadata={
-            "publication_id": ref.publication_id,
-            **metadata,
-        },
-    )
-
-
-def scrape_who(
-    *,
-    documents: int | None,
-    link_mode: LinkMode = LinkMode.KEEP,
-    url: str | None = None,
-) -> ScrapeRun:
-    """Scrape WHO documents from a URL or guideline listing pages.
-
-    Args:
-        documents: Number of documents to scrape. Ignored when `url` is set.
-            When unset, WHO listing pages are fetched until a page returns no
-            items (default: None).
-        link_mode: Whether links are kept as markdown links or stripped to their
-            visible text (default: LinkMode.KEEP).
-        url: WHO publication URL to scrape as a single document (default: None).
-    """
-    if url is not None:
-
-        def scrape_url() -> Iterable[ScrapedDocument]:
-            with default_client() as client:
-                yield scrape_publication(client, publication_ref_from_url(url), link_mode=link_mode)
-
-        return ScrapeRun(documents=scrape_url(), total=1)
-
-    with default_client() as client:
-        first_page = list_publications(client, page=1)
-    total = first_page.total if documents is None or first_page.total is None else min(documents, first_page.total)
-    return ScrapeRun(
-        total=total,
-        documents=scrape_listing_documents(
-            documents=documents,
-            client_factory=default_client,
-            first_page_items=first_page.refs,
-            list_page=lambda client, page: list_publications(client, page).refs,
-            scrape_item=lambda client, ref: scrape_publication(client, ref, link_mode=link_mode),
-            document_delay_seconds=DOCUMENT_DELAY_SECONDS,
-        ),
-    )
-
-
-__all__ = [
-    "BASE_URL",
-    "CONTENT_SCOPE",
-    "DOCUMENT_DELAY_SECONDS",
-    "GUIDELINES_LISTING_URL",
-    "WHO_ATTRIBUTION",
-    "WHO_LICENSE",
-    "WhoFetchError",
-    "WhoListingPage",
-    "WhoPublicationRef",
-    "build_publication_text",
-    "list_publications",
-    "publication_ref_from_url",
-    "scrape_publication",
-    "scrape_who",
-]
diff --git a/datasets/test/fixtures/who_listing_api.json b/datasets/test/fixtures/who_listing_api.json
deleted file mode 100644
index a4392d3b..00000000
--- a/datasets/test/fixtures/who_listing_api.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-  "@odata.context": "https://www.who.int/api/hubs/$metadata#publications(Title,ItemDefaultUrl,FormatedDate,Tag,DownloadUrl)",
-  "@odata.count": 356,
-  "value": [
-    {
-      "ItemDefaultUrl": "/9789240121805",
-      "Title": "Guidelines for the prevention of bloodstream infections and other infections associated with the use of intravascular catheters: part 2: central venous catheters",
-      "FormatedDate": "28 May 2026",
-      "Tag": "Guideline",
-      "DownloadUrl": "https://iris.who.int/server/api/core/bitstreams/f750f24d-c0c2-425c-85fd-ec310d2ce994/content"
-    },
-    {
-      "ItemDefaultUrl": "/9789240121744",
-      "Title": "WHO guideline for screening and treatment of cervical pre-cancer lesions for cervical cancer prevention",
-      "FormatedDate": "8 May 2026",
-      "Tag": "Guideline",
-      "DownloadUrl": "https://iris.who.int/server/api/core/bitstreams/32214b73-0e95-4243-9e83-617948510dcd/content"
-    }
-  ]
-}
diff --git a/datasets/test/fixtures/who_publication_overview.html b/datasets/test/fixtures/who_publication_overview.html
deleted file mode 100644
index b6f16f29..00000000
--- a/datasets/test/fixtures/who_publication_overview.html
+++ /dev/null
@@ -1,17 +0,0 @@
-<html>
-  <article class="dynamic-content dynamic-content__article dynamic-content__publication">
-    <h1>Guidelines for the prevention of bloodstream infections and other infections associated with the use of intravascular catheters: part 2: central venous catheters</h1>
-    <div class="dynamic-content__data">
-      <div class="dynamic-content__date">28 May 2026</div>
-      <div class="dynamic-content__tag"><span>&nbsp;&#124;&nbsp;</span>Guideline</div>
-    </div>
-    <section class="dynamic-content__section dynamic-content__container">
-      <a href="https://iris.who.int/server/api/core/bitstreams/f750f24d-c0c2-425c-85fd-ec310d2ce994/content">Download (1.2 MB)</a>
-      <h3>Overview</h3>
-      <p>These WHO guidelines provide evidence-based recommendations for the prevention of bloodstream infections (BSIs) and other infections associated with the use of central venous catheters (CVCs) across health care settings.</p>
-      <h3>WHO Team</h3>
-      <p>Infection Prevention and Control (IPC)</p>
-      <p><strong>ISBN:</strong> 978-92-4-012180-5</p>
-    </section>
-  </article>
-</html>
diff --git a/datasets/test/test_scraping_cli.py b/datasets/test/test_scraping_cli.py
index bc4eccd2..44f714fa 100644
--- a/datasets/test/test_scraping_cli.py
+++ b/datasets/test/test_scraping_cli.py
@@ -70,15 +70,11 @@ def fake_scrape_documents(
         documents: int | None,
         link_mode: LinkMode,
         url: str | None = None,
-        include_archived: bool = False,
-        include_in_development: bool = False,
     ) -> ScrapeRun:
         assert source is ScraperSource.NICE
         assert documents == 3
         assert link_mode is LinkMode.STRIP
         assert url is None
-        assert include_archived is False
-        assert include_in_development is False
         return ScrapeRun([_document()], total=3)
 
     monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_documents", fake_scrape_documents)
@@ -109,8 +105,6 @@ def fake_scrape_documents(
         documents: int | None,
         link_mode: LinkMode,
         url: str | None = None,
-        include_archived: bool = False,
-        include_in_development: bool = False,
     ) -> ScrapeRun:
         return ScrapeRun([_document()], total=None)
 
@@ -143,8 +137,6 @@ def fake_scrape_documents(
         documents: int | None,
         link_mode: LinkMode,
         url: str | None = None,
-        include_archived: bool = False,
-        include_in_development: bool = False,
     ) -> ScrapeRun:
         return ScrapeRun([_document()], total=7)
 
@@ -212,15 +204,11 @@ def fake_scrape_documents(
         documents: int | None,
         link_mode: LinkMode,
         url: str | None = None,
-        include_archived: bool = False,
-        include_in_development: bool = False,
     ) -> ScrapeRun:
         assert source is ScraperSource.ALL
         assert documents is None
         assert link_mode is LinkMode.KEEP
         assert url is None
-        assert include_archived is False
-        assert include_in_development is False
         return ScrapeRun([_document()], total=12)
 
     monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_documents", fake_scrape_documents)
@@ -231,115 +219,6 @@ def fake_scrape_documents(
     assert json.loads(result.stdout.splitlines()[0])["external_id"] == "nice-ng1"
 
 
-def test_cli_run_accepts_idsa_status_flags(monkeypatch: pytest.MonkeyPatch) -> None:
-    """The CLI forwards IDSA status inclusion flags."""
-    runner = CliRunner()
-
-    def fake_scrape_documents(
-        source: ScraperSource,
-        *,
-        documents: int | None,
-        link_mode: LinkMode,
-        url: str | None = None,
-        include_archived: bool = False,
-        include_in_development: bool = False,
-    ) -> ScrapeRun:
-        assert source is ScraperSource.IDSA
-        assert documents == 2
-        assert link_mode is LinkMode.KEEP
-        assert url is None
-        assert include_archived is True
-        assert include_in_development is True
-        return ScrapeRun([_idsa_document()], total=2)
-
-    monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_documents", fake_scrape_documents)
-
-    result = runner.invoke(
-        app,
-        ["--source", "idsa", "--documents", "2", "--include-archived", "--include-in-development"],
-    )
-
-    assert result.exit_code == 0
-    assert json.loads(result.stdout.splitlines()[0])["external_id"] == "idsa-current-guideline"
-    assert "scraped 1 documents from idsa" in result.stderr
-
-
-def test_scrape_documents_dispatches_idsa(monkeypatch: pytest.MonkeyPatch) -> None:
-    """The scrape dispatcher calls the IDSA scraper for the IDSA source."""
-
-    def fake_scrape_idsa(
-        *,
-        documents: int | None,
-        link_mode: LinkMode,
-        url: str | None,
-        include_archived: bool,
-        include_in_development: bool,
-    ) -> ScrapeRun:
-        assert documents == 1
-        assert link_mode is LinkMode.STRIP
-        assert url == "https://www.idsociety.org/practice-guideline/current-guideline/"
-        assert include_archived is True
-        assert include_in_development is False
-        return ScrapeRun([_idsa_document()], total=1)
-
-    monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_idsa", fake_scrape_idsa)
-
-    from amfv_datasets.scraping.cli import scrape_documents
-
-    scrape_run = scrape_documents(
-        ScraperSource.IDSA,
-        documents=1,
-        link_mode=LinkMode.STRIP,
-        url="https://www.idsociety.org/practice-guideline/current-guideline/",
-        include_archived=True,
-    )
-
-    assert list(scrape_run.documents) == [_idsa_document()]
-
-
-def test_scrape_documents_all_includes_idsa(monkeypatch: pytest.MonkeyPatch) -> None:
-    """The all source combines NICE and IDSA scrape runs."""
-
-    def fake_scrape_nice(
-        *,
-        documents: int | None,
-        link_mode: LinkMode,
-        url: str | None,
-    ) -> ScrapeRun:
-        assert documents == 1
-        assert link_mode is LinkMode.KEEP
-        assert url is None
-        return ScrapeRun([_document()], total=1)
-
-    def fake_scrape_idsa(
-        *,
-        documents: int | None,
-        link_mode: LinkMode,
-        url: str | None,
-        include_archived: bool,
-        include_in_development: bool,
-    ) -> ScrapeRun:
-        assert documents == 1
-        assert link_mode is LinkMode.KEEP
-        assert url is None
-        assert include_archived is False
-        assert include_in_development is False
-        return ScrapeRun([_idsa_document()], total=1)
-
-    monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_nice", fake_scrape_nice)
-    monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_idsa", fake_scrape_idsa)
-
-    from amfv_datasets.scraping.cli import scrape_documents
-
-    scrape_run = scrape_documents(ScraperSource.ALL, documents=1, link_mode=LinkMode.KEEP)
-
-    assert scrape_run.total == 2
-    assert [document.external_id for document in scrape_run.documents] == [
-        "nice-ng1",
-        "idsa-current-guideline",
-    ]
-
-
 def _document() -> ScrapedDocument:
     return ScrapedDocument(
         source="nice",
@@ -351,17 +230,6 @@ def _document() -> ScrapedDocument:
     )
 
 
-def _idsa_document() -> ScrapedDocument:
-    return ScrapedDocument(
-        source="idsa",
-        external_id="idsa-current-guideline",
-        title="Current Guideline",
-        url="https://www.idsociety.org/practice-guideline/current-guideline/",
-        content="content",
-        metadata={"slug": "current-guideline"},
-    )
-
-
 class _TextSink:
     def __init__(self) -> None:
         self.value = ""
diff --git a/datasets/test/test_scraping_cps.py b/datasets/test/test_scraping_cps.py
deleted file mode 100644
index c50ce14c..00000000
--- a/datasets/test/test_scraping_cps.py
+++ /dev/null
@@ -1,153 +0,0 @@
-"""Tests for Canadian Paediatric Society statement scraping helpers."""
-
-import httpx
-import pytest
-
-from amfv_datasets.scraping.cps import (
-    BASE_URL,
-    STATEMENTS_URL,
-    CpsFetchError,
-    CpsStatementRef,
-    list_statements,
-    listing_page_url,
-    scrape_statement,
-    statement_ref_from_url,
-)
-
-
-def test_listing_page_url_uses_cps_offsets() -> None:
-    """One-based pages map to the CPS P30 offset convention."""
-    assert listing_page_url(1) == STATEMENTS_URL
-    assert listing_page_url(2) == f"{STATEMENTS_URL}/P10"
-    assert listing_page_url(4) == f"{STATEMENTS_URL}/P30"
-
-
-def test_list_statements_discovers_unique_position_pages() -> None:
-    """The index yields unique position statements and ignores unrelated links."""
-    html_text = """
-    <html><div class="cell main-body">
-      <div class="stmt-title"><a href="/en/documents/position/acute-asthma">Acute asthma</a></div>
-      <div class="stmt-title"><a href="https://cps.ca/documents/position/acute-asthma">Duplicate asthma</a></div>
-      <div class="stmt-title"><a href="/en/documents/position/newborn-glucose">Newborn glucose</a></div>
-      <a href="/en/documents/about-position-statements">About statements</a>
-    </div><footer><a href="/en/documents/position/footer-link">Footer</a></footer></html>
-    """
-
-    def handler(request: httpx.Request) -> httpx.Response:
-        assert str(request.url) == STATEMENTS_URL
-        return httpx.Response(200, text=html_text)
-
-    with httpx.Client(transport=httpx.MockTransport(handler)) as client:
-        assert list_statements(client, 1) == [
-            CpsStatementRef("acute-asthma", "Acute asthma", f"{BASE_URL}/en/documents/position/acute-asthma"),
-            CpsStatementRef(
-                "newborn-glucose",
-                "Newborn glucose",
-                f"{BASE_URL}/en/documents/position/newborn-glucose",
-            ),
-        ]
-
-
-def test_statement_ref_from_url_normalizes_supported_routes() -> None:
-    """English and language-neutral CPS routes become canonical HTTPS URLs."""
-    assert statement_ref_from_url("http://www.cps.ca/documents/position/Acute-Asthma?print=1#dose") == (
-        CpsStatementRef(
-            "Acute-Asthma",
-            "Acute Asthma",
-            "https://cps.ca/en/documents/position/Acute-Asthma",
-        )
-    )
-
-
-def test_statement_ref_from_url_rejects_non_statement_pages() -> None:
-    """CPS navigation pages cannot be mistaken for clinical statements."""
-    with pytest.raises(CpsFetchError):
-        statement_ref_from_url("https://cps.ca/en/documents")
-
-
-def test_scrape_statement_preserves_clinical_structure_and_metadata() -> None:
-    """Statement content is normalized while navigation and sharing chrome are removed."""
-    html_text = """
-    <html>
-      <head><title>Fallback | Canadian Paediatric Society</title></head>
-      <body>
-        <header><h1>Site heading</h1></header>
-        <main id="main-content">
-          <div class="statement-wrapper">
-            <div class="breadcrumb">Home / Documents</div>
-            <p>Position statement</p>
-            <h1>Management of well-appearing febrile young infants</h1>
-            <p>Posted: Oct 27, 2023 | Reaffirmed: Jan 12, 2026 | Updated: May 27, 2026</p>
-            <h2>Recommendations</h2>
-            <ul><li>Assess the infant:<ul><li>Check vital signs.</li></ul></li></ul>
-            <p>Use the <a href="/en/tools/risk-calculator">risk calculator</a>.</p>
-            <p>Current citation<a class="reference" href="#ref1">[1]</a>.</p>
-            <p>Legacy citation<sup>[<a class="reference" href="#ref2">2</a>]</sup>.</p>
-            <p>Maternal fever above 38<sup>o</sup>C and counts of 10<sup>9</sup>/L require attention.</p>
-            <p><a href="/documents/full-statement.pdf">Full statement PDF</a>
-              <img src="/assets/img/file-pdf.svg" alt="PDF icon"></p>
-            <p><a href="/empty-target"></a></p>
-            <table><tr><th>Age</th><th>Action</th></tr><tr><td>0-28 days</td><td>Investigate</td></tr></table>
-            <h2><strong>Algorithm</strong></h2>
-            <p><img src="/uploads/flowcharts/febrile-infant.png" alt=""></p>
-            <h2>References</h2>
-            <div class="reference-print-style"><ol>
-              <li><a name="ref1"></a>Reference one.</li>
-              <li><a name="ref2"></a>Reference two.</li>
-            </ol></div>
-            <a href="/en/education/test-your-knowledge"><img src="/assets/img/test.png" alt="Quiz"></a>
-            <div class="share-tools">Share this statement</div>
-            <aside class="related-content">Related news</aside>
-            <script>window.track = true;</script>
-          </div>
-        </main>
-        <footer>Contact CPS</footer>
-      </body>
-    </html>
-    """
-    ref = CpsStatementRef(
-        "febrile-young-infants",
-        "Listing title",
-        f"{BASE_URL}/en/documents/position/febrile-young-infants",
-    )
-
-    def handler(request: httpx.Request) -> httpx.Response:
-        assert str(request.url) == ref.page_url
-        return httpx.Response(200, text=html_text)
-
-    with httpx.Client(transport=httpx.MockTransport(handler)) as client:
-        document = scrape_statement(client, ref)
-
-    assert document.source == "cps"
-    assert document.external_id == "cps-febrile-young-infants"
-    assert document.title == "Management of well-appearing febrile young infants"
-    assert document.section_count == 3
-    assert document.metadata == {
-        "slug": "febrile-young-infants",
-        "posted": "Oct 27, 2023",
-        "reaffirmed": "Jan 12, 2026",
-        "updated": "May 27, 2026",
-    }
-    assert "- Assess the infant:\n  - Check vital signs." in document.content
-    assert "[risk calculator](https://cps.ca/en/tools/risk-calculator)" in document.content
-    assert "Current citation[1](https://cps.ca/en/documents/position/febrile-young-infants#ref1)." in document.content
-    assert "Legacy citation[2](https://cps.ca/en/documents/position/febrile-young-infants#ref2)." in document.content
-    assert "38<sup>o</sup>C" in document.content
-    assert "10<sup>9</sup>/L" in document.content
-    assert "# Management of well-appearing" not in document.content
-    assert "## Algorithm" in document.content
-    assert "## **Algorithm**" not in document.content
-    assert "| Age | Action |" in document.content
-    assert "![](https://cps.ca/uploads/flowcharts/febrile-infant.png)" in document.content
-    assert "## References" in document.content
-    assert "1. Reference one." in document.content
-    assert "2. Reference two." in document.content
-    assert "[Full statement PDF](https://cps.ca/documents/full-statement.pdf)" in document.content
-    assert "file-pdf.svg" not in document.content
-    assert "empty-target" not in document.content
-    assert "Share this statement" not in document.content
-    assert "Related news" not in document.content
-    assert "test-your-knowledge" not in document.content
-    assert "assets/img/test.png" not in document.content
-    assert "window.track" not in document.content
-    assert "Contact CPS" not in document.content
diff --git a/datasets/test/test_scraping_idsa.py b/datasets/test/test_scraping_idsa.py
deleted file mode 100644
index 2c6073f7..00000000
--- a/datasets/test/test_scraping_idsa.py
+++ /dev/null
@@ -1,258 +0,0 @@
-"""Tests for IDSA scraping helpers."""
-
-import httpx
-import pytest
-
-from amfv_datasets.scraping.html import LinkMode
-from amfv_datasets.scraping.idsa import (
-    BASE_URL,
-    LISTING_URL,
-    IDSAFetchError,
-    IDSAGuidelineRef,
-    idsa_ref_from_url,
-    list_practice_guidelines,
-    scrape_guideline,
-    scrape_idsa,
-)
-
-_LISTING_CASES = (
-    ("current-guideline", "Current Guideline", 2024, ("Current",)),
-    ("current-endorsed-guideline", "Current Endorsed Guideline", 2023, ("Current", "Endorsed")),
-    ("endorsed-guideline", "Endorsed Guideline", 2022, ("Endorsed",)),
-    ("archived-guideline", "Archived Guideline", 2021, ("Archived",)),
-    ("development-guideline", "Development Guideline", 2020, ("In Development",)),
-    ("archived-development-guideline", "Archived Development Guideline", 2019, ("Archived", "In Development")),
-)
-
-
-def test_list_practice_guidelines_parses_listing_fields() -> None:
-    """The IDSA listing parser normalizes title, URL, year, and statuses."""
-    listing = list_practice_guidelines(_listing_client())
-
-    assert listing.total == 2
-    assert listing.refs == [
-        IDSAGuidelineRef(
-            title="Current Guideline",
-            slug="current-guideline",
-            page_url="https://www.idsociety.org/practice-guideline/current-guideline/",
-            year=2024,
-            statuses=("Current",),
-        ),
-        IDSAGuidelineRef(
-            title="Current Endorsed Guideline",
-            slug="current-endorsed-guideline",
-            page_url="https://www.idsociety.org/practice-guideline/current-endorsed-guideline/",
-            year=2023,
-            statuses=("Current", "Endorsed"),
-        ),
-    ]
-
-
-@pytest.mark.parametrize(
-    ("kwargs", "expected_slugs"),
-    [
-        ({}, ["current-guideline", "current-endorsed-guideline"]),
-        (
-            {"include_archived": True},
-            ["current-guideline", "current-endorsed-guideline", "archived-guideline"],
-        ),
-        (
-            {"include_in_development": True},
-            ["current-guideline", "current-endorsed-guideline", "development-guideline"],
-        ),
-        (
-            {"include_archived": True, "include_in_development": True},
-            [
-                "current-guideline",
-                "current-endorsed-guideline",
-                "archived-guideline",
-                "development-guideline",
-                "archived-development-guideline",
-            ],
-        ),
-    ],
-    ids=["default", "include-archived", "include-in-development", "include-both"],
-)
-def test_list_practice_guidelines_filters_statuses(kwargs: dict[str, bool], expected_slugs: list[str]) -> None:
-    """Status flags preserve the intended IDSA listing inclusion policy."""
-    listing = list_practice_guidelines(_listing_client(), **kwargs)
-
-    assert [ref.slug for ref in listing.refs] == expected_slugs
-
-
-def test_idsa_ref_from_url_normalizes_practice_guideline_url() -> None:
-    """IDSA practice guideline URLs are normalized to canonical refs."""
-    assert idsa_ref_from_url("https://www.idsociety.org/practice-guideline/Current-Guideline/?utm=1") == (
-        IDSAGuidelineRef(
-            title="Current Guideline",
-            slug="current-guideline",
-            page_url="https://www.idsociety.org/practice-guideline/current-guideline/",
-            year=None,
-            statuses=(),
-        )
-    )
-
-
-@pytest.mark.parametrize(
-    "url",
-    [
-        "https://example.com/practice-guideline/current-guideline/",
-        "https://www.idsociety.org/news/current-guideline/",
-    ],
-    ids=["wrong-domain", "wrong-path"],
-)
-def test_idsa_ref_from_url_rejects_non_guideline_urls(url: str) -> None:
-    """Only IDSA practice guideline URLs are accepted."""
-    with pytest.raises(IDSAFetchError):
-        idsa_ref_from_url(url)
-
-
-def test_scrape_guideline_extracts_content_and_link_metadata() -> None:
-    """Guideline page content is converted to markdown and noisy UI is stripped."""
-    document = scrape_guideline(_guideline_client(_guideline_html()), _ref())
-
-    assert document.title == "Current Guideline"
-    assert document.section_count == 3
-    assert "# Current Guideline" in document.content
-    assert "## Abstract" in document.content
-    assert "## Recommendations" in document.content
-    assert "[Download PDF](https://academic.oup.com/example.pdf)" in document.content
-    assert "Recommendation text with [evidence](https://doi.org/10.1093/cid/example)." in document.content
-    assert "Back to top" not in document.content
-    assert "Table of Contents" not in document.content
-    assert "https://www.idsociety.org#abstract" not in document.content
-    assert document.metadata["external_links"] == [
-        "https://academic.oup.com/example.pdf",
-        "https://doi.org/10.1093/cid/example",
-    ]
-    assert document.metadata["pdf_links"] == ["https://academic.oup.com/example.pdf"]
-    assert document.metadata["content_length_chars"] == len(document.content)
-    assert document.metadata["quality_flags"] == ["short_content"]
-
-
-def test_scrape_guideline_strips_links_when_requested() -> None:
-    """The IDSA scraper honors the shared link-mode option."""
-    document = scrape_guideline(_guideline_client(_guideline_html()), _ref(), link_mode=LinkMode.STRIP)
-
-    assert "Recommendation text with evidence." in document.content
-    assert "[evidence]" not in document.content
-
-
-def test_scrape_idsa_returns_normalized_document() -> None:
-    """The IDSA scraper returns normalized scraped documents."""
-    transport = httpx.MockTransport(_scrape_idsa_handler)
-
-    class ClientFactory:
-        def __call__(self) -> httpx.Client:
-            return httpx.Client(transport=transport)
-
-    with pytest.MonkeyPatch.context() as monkeypatch:
-        monkeypatch.setattr("amfv_datasets.scraping.idsa.default_client", ClientFactory())
-        scrape_run = scrape_idsa(documents=1)
-        documents = list(scrape_run.documents)
-
-    assert scrape_run.total == 1
-    assert len(documents) == 1
-    assert documents[0].source == "idsa"
-    assert documents[0].external_id == "idsa-current-guideline"
-    assert documents[0].metadata["statuses"] == ["Current"]
-    assert documents[0].metadata["year"] == 2024
-    assert documents[0].metadata["slug"] == "current-guideline"
-    assert documents[0].metadata["listing_url"] == LISTING_URL
-
-
-@pytest.mark.parametrize(
-    ("paragraph", "expected_flags"),
-    [
-        ("Useful abstract.", ["short_content"]),
-        ("Recommendation text. " * 700, []),
-    ],
-    ids=["short", "long"],
-)
-def test_scrape_guideline_sets_content_quality_flags(paragraph: str, expected_flags: list[str]) -> None:
-    """Short content is flagged while long content is left unflagged."""
-    document = scrape_guideline(_guideline_client(_guideline_html(paragraph=paragraph)), _ref())
-
-    assert document.metadata["content_length_chars"] == len(document.content)
-    assert document.metadata["quality_flags"] == expected_flags
-
-
-def _listing_client() -> httpx.Client:
-    def handler(request: httpx.Request) -> httpx.Response:
-        assert str(request.url) == LISTING_URL
-        return httpx.Response(200, text=_listing_html())
-
-    return httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL)
-
-
-def _guideline_client(content_html: str) -> httpx.Client:
-    def handler(request: httpx.Request) -> httpx.Response:
-        assert str(request.url) == "https://www.idsociety.org/practice-guideline/current-guideline/"
-        return httpx.Response(200, text=content_html)
-
-    return httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL)
-
-
-def _scrape_idsa_handler(request: httpx.Request) -> httpx.Response:
-    if str(request.url) == LISTING_URL:
-        return httpx.Response(200, text=_listing_html())
-    if str(request.url) == "https://www.idsociety.org/practice-guideline/current-guideline/":
-        return httpx.Response(200, text=_guideline_html())
-    raise AssertionError(f"Unexpected URL: {request.url}")
-
-
-def _listing_html() -> str:
-    items = "\n".join(
-        _listing_item(slug=slug, title=title, year=year, statuses=statuses)
-        for slug, title, year, statuses in _LISTING_CASES
-    )
-    return f'<html><div class="alpha-listing"><ul class="list-pages">{items}</ul></div></html>'
-
-
-def _listing_item(*, slug: str, title: str, year: int, statuses: tuple[str, ...]) -> str:
-    categories = "".join(f'<li class="category-dot">{status}</li>' for status in statuses)
-    return f"""
-        <li>
-          <ul class="list-pages__categories">{categories}</ul>
-          <a class="list-pages__link" href="/practice-guideline/{slug}/">{title}</a>
-          <span class="list-pages__year">{year}</span>
-        </li>
-    """
-
-
-def _guideline_html(*, title: str = "Current Guideline", paragraph: str = "Useful abstract.") -> str:
-    return f"""
-        <html>
-          <title>{title} | IDSA</title>
-          <div class="standardpage-col-left">
-            <nav>Navigation noise</nav>
-            <p>Intro column noise.</p>
-          </div>
-          <div class="standardpage-col-left">
-            <p><a href="https://academic.oup.com/example.pdf">Download PDF</a></p>
-            <h1>{title}</h1>
-            <h3>Table of Contents</h3>
-            <ul>
-              <li><a href="#abstract">Abstract</a></li>
-              <li><a href="#recommendations">Recommendations</a></li>
-            </ul>
-            <p class="social">Share this page</p>
-            <p>Published January 1, 2024</p>
-            <h2>Abstract</h2>
-            <p>{paragraph}</p>
-            <h2>Recommendations</h2>
-            <p>Recommendation text with <a href="https://doi.org/10.1093/cid/example">evidence</a>.</p>
-            <p>Back to top</p>
-          </div>
-        </html>
-    """
-
-
-def _ref() -> IDSAGuidelineRef:
-    return IDSAGuidelineRef(
-        title="Current Guideline",
-        slug="current-guideline",
-        page_url="https://www.idsociety.org/practice-guideline/current-guideline/",
-        year=2024,
-        statuses=("Current",),
-    )
diff --git a/datasets/test/test_scraping_pubmed.py b/datasets/test/test_scraping_pubmed.py
deleted file mode 100644
index 16deaaba..00000000
--- a/datasets/test/test_scraping_pubmed.py
+++ /dev/null
@@ -1,478 +0,0 @@
-"""Tests for PubMed scraping helpers."""
-
-import json
-
-import httpx
-import pytest
-
-from amfv_datasets.scraping import base
-from amfv_datasets.scraping.html import LinkMode
-from amfv_datasets.scraping.pubmed import (
-    LISTING_PAGE_SIZE,
-    PubMedArticleRef,
-    PubMedFetchError,
-    build_pubmed_article_text,
-    list_pubmed_guidelines,
-    pubmed_ref_from_url,
-    scrape_pubmed,
-    scrape_pubmed_article,
-)
-
-_BODY = """
-<body>
-  <sec><title>Scope and purpose</title>
-    <p>This guideline covers <italic>adult</italic> hypertension.</p>
-    <sec><title>Guideline objectives</title><p>Reduce cardiovascular risk.</p></sec>
-  </sec>
-  <sec><title>Recommendations</title>
-    <p>Treat to target <xref ref-type="bibr" rid="r1">[1]</xref> .</p>
-    <table-wrap><label>Table 1</label>
-      <caption><title>Thresholds</title></caption>
-      <table>
-        <thead><tr><th>Stage</th><th>Threshold</th></tr></thead>
-        <tbody><tr><td>Stage 2</td><td>140/90 mmHg</td></tr></tbody>
-      </table>
-    </table-wrap>
-    <list list-type="order"><list-item><p>Measure blood pressure.</p></list-item></list>
-  </sec>
-  <sec><title>Treatment algorithm</title>
-    <fig id="g1-F1"><label>Figure 1</label>
-      <caption><p>Stepped care algorithm for resistant hypertension.</p></caption>
-      <graphic xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="g1f1.jpg"/>
-    </fig>
-  </sec>
-  <supplementary-material content-type="local-data" id="MOESM1">
-    <media xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="12345_MOESM1_ESM.docx"/>
-    <caption><p>Additional file 1: Table S1. Evidence profile for each recommendation.</p></caption>
-  </supplementary-material>
-  <ref-list><ref id="r1"><mixed-citation>Smith J. Irrelevant citation.</mixed-citation></ref></ref-list>
-</body>
-"""
-
-_LICENSE_ALI = (
-    "<license><ali:license_ref>https://creativecommons.org/licenses/by/4.0/</ali:license_ref>"
-    "<license-p>Open access.</license-p></license>"
-)
-_LICENSE_EXT_LINK = (
-    "<license><license-p>Distributed under the "
-    '<ext-link ext-link-type="uri" xlink:href="https://creativecommons.org/licenses/by-nc/4.0/">'
-    "CC BY-NC</ext-link> license.</license-p></license>"
-)
-_LICENSE_TEXT_ONLY = (
-    "<license><license-p>Distributed under "
-    "https://creativecommons.org/licenses/by-nc-nd/4.0/ terms.</license-p></license>"
-)
-# 11.7% of the open-access subset carries publisher terms instead of a CC
-# license, and they are not equivalent to each other.
-_LICENSE_NON_CC_PERMISSIVE = (
-    '<license license-type="open-access"><license-p>This article is made available via the '
-    "PMC Open Access Subset for unrestricted re-use and analyses in any form or by any means "
-    "with acknowledgement of the original source.</license-p></license>"
-)
-_LICENSE_NON_CC_RESERVED = (
-    "<copyright-statement>© 2020 Elsevier Masson SAS. All rights reserved.</copyright-statement>"
-    "<license><license-p>Since January 2020 Elsevier has created a COVID-19 resource centre "
-    "with free information.</license-p></license>"
-)
-
-
-def _jats(*, body: str = _BODY, license_block: str = _LICENSE_ALI, abstract: str = "") -> str:
-    """Build a minimal but structurally faithful PMC JATS response."""
-    return (
-        '<?xml version="1.0"?>'
-        '<pmc-articleset><article article-type="review-article"'
-        ' xmlns:xlink="http://www.w3.org/1999/xlink"'
-        ' xmlns:ali="http://www.niso.org/schemas/ali/1.0/">'
-        "<front>"
-        "<journal-meta><journal-title>Journal of Guidelines</journal-title></journal-meta>"
-        "<article-meta>"
-        '<article-id pub-id-type="pmid">42476581</article-id>'
-        '<article-id pub-id-type="pmcid">PMC13384732</article-id>'
-        '<article-id pub-id-type="doi">10.1093/example.1</article-id>'
-        "<title-group><article-title>Hypertension guideline</article-title></title-group>"
-        f"<permissions>{license_block}</permissions>"
-        f"{abstract}"
-        "</article-meta></front>"
-        f"{body}"
-        "</article></pmc-articleset>"
-    )
-
-
-def _article_client(xml: str) -> httpx.Client:
-    def handler(request: httpx.Request) -> httpx.Response:
-        assert request.url.path.endswith("efetch.fcgi")
-        assert request.url.params["db"] == "pmc"
-        # NCBI's usage policy asks every request to name the calling tool.
-        assert request.url.params["tool"] == "amfv"
-        return httpx.Response(200, content=xml.encode())
-
-    return httpx.Client(transport=httpx.MockTransport(handler))
-
-
-def _listing_handler(
-    pages: dict[int, list[str]],
-    *,
-    total: int,
-    summaries: dict[str, dict] | None = None,
-    seen_offsets: list[int] | None = None,
-):
-    """Serve esearch/esummary/efetch for a fake corpus keyed by PMID."""
-
-    def handler(request: httpx.Request) -> httpx.Response:
-        path = request.url.path
-        params = request.url.params
-        if path.endswith("esearch.fcgi"):
-            offset = int(params["retstart"])
-            if seen_offsets is not None:
-                seen_offsets.append(offset)
-            page = offset // LISTING_PAGE_SIZE + 1
-            return httpx.Response(
-                200,
-                text=json.dumps({"esearchresult": {"count": str(total), "idlist": pages.get(page, [])}}),
-            )
-        if path.endswith("esummary.fcgi"):
-            uids = params["id"].split(",")
-            result = {"uids": uids}
-            for uid in uids:
-                result[uid] = (summaries or {}).get(
-                    uid,
-                    {
-                        "title": f"Guideline {uid}",
-                        "fulljournalname": "Journal of Guidelines",
-                        "pubdate": "2026 Jan 1",
-                        "articleids": [
-                            {"idtype": "pubmed", "value": uid},
-                            {"idtype": "doi", "value": f"10.1000/{uid}"},
-                            {"idtype": "pmc", "value": f"PMC{uid}"},
-                        ],
-                    },
-                )
-            return httpx.Response(200, text=json.dumps({"result": result}))
-        pmcid = params["id"]
-        if pmcid == "999":
-            # PMC holds the record but no full text was deposited.
-            return httpx.Response(200, content=_jats(body="").encode())
-        return httpx.Response(200, content=_jats().encode())
-
-    return handler
-
-
-def test_list_pubmed_guidelines_resolves_pmids_to_refs_with_pmcids() -> None:
-    """One esearch plus one esummary yields refs carrying full citation metadata."""
-    client = httpx.Client(transport=httpx.MockTransport(_listing_handler({1: ["111", "222"]}, total=2)))
-
-    listing = list_pubmed_guidelines(client)
-
-    assert listing.total == 2
-    assert [ref.pmid for ref in listing.refs] == ["111", "222"]
-    assert listing.refs[0].pmcid == "PMC111"
-    assert listing.refs[0].title == "Guideline 111"
-    assert listing.refs[0].doi == "10.1000/111"
-    assert listing.refs[0].page_url == "https://pubmed.ncbi.nlm.nih.gov/111/"
-
-
-def test_list_pubmed_guidelines_pages_by_numeric_offset() -> None:
-    """Page number maps onto `retstart`, which is what makes paging work at all."""
-    seen: list[int] = []
-    client = httpx.Client(transport=httpx.MockTransport(_listing_handler({3: ["333"]}, total=900, seen_offsets=seen)))
-
-    listing = list_pubmed_guidelines(client, page=3)
-
-    assert seen == [2 * LISTING_PAGE_SIZE]
-    assert [ref.pmid for ref in listing.refs] == ["333"]
-
-
-def test_list_pubmed_guidelines_returns_no_refs_past_the_end_of_the_listing() -> None:
-    """An empty page terminates the scrape loop rather than raising."""
-    client = httpx.Client(transport=httpx.MockTransport(_listing_handler({}, total=2)))
-
-    listing = list_pubmed_guidelines(client, page=99)
-
-    assert listing.refs == []
-    assert listing.total == 2
-
-
-def test_list_pubmed_guidelines_skips_records_with_no_pmc_copy() -> None:
-    """The open-access filter should guarantee a PMC id; a record without one is dropped."""
-    summaries = {
-        "111": {"title": "No full text", "articleids": [{"idtype": "pubmed", "value": "111"}]},
-    }
-    client = httpx.Client(
-        transport=httpx.MockTransport(_listing_handler({1: ["111", "222"]}, total=2, summaries=summaries))
-    )
-
-    listing = list_pubmed_guidelines(client)
-
-    assert [ref.pmid for ref in listing.refs] == ["222"]
-
-
-def test_build_pubmed_article_text_maps_jats_sections_to_nested_headings() -> None:
-    """`<sec>` nesting depth becomes markdown heading depth."""
-    content, section_count, title, metadata = build_pubmed_article_text(
-        _article_client(_jats()),
-        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
-        link_mode=LinkMode.STRIP,
-    )
-
-    assert "## Scope and purpose" in content
-    assert "### Guideline objectives" in content
-    assert "## Recommendations" in content
-    assert title == "Hypertension guideline"
-    assert section_count == 3
-    assert metadata["journal"] == "Journal of Guidelines"
-    assert metadata["article_type"] == "review-article"
-    assert metadata["content_length_chars"] == len(content)
-
-
-def test_build_pubmed_article_text_keeps_clinical_tables_and_drops_citation_pointers() -> None:
-    """Recommendation thresholds must survive; `xref` markers and reference lists must not."""
-    content, _, _, _ = build_pubmed_article_text(
-        _article_client(_jats()),
-        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
-    )
-
-    assert "140/90 mmHg" in content
-    assert "Measure blood pressure" in content
-    # The `[1]` pointer and the reference list it points at are citation
-    # apparatus, not clinical content.
-    assert "[1]" not in content
-    assert "Irrelevant citation" not in content
-    # Unwrapping the pointer must not leave " ." behind.
-    assert "Treat to target ." not in content
-    assert "Treat to target." in content
-
-
-def test_build_pubmed_article_text_records_figures_it_cannot_link() -> None:
-    """The image URL is not derivable from the API, so keep the filename for a later pass."""
-    content, _, _, metadata = build_pubmed_article_text(
-        _article_client(_jats()),
-        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
-    )
-
-    assert metadata["figures"] == [
-        {
-            "id": "g1-F1",
-            "file": "g1f1.jpg",
-            "label": "Figure 1",
-            "caption": "Stepped care algorithm for resistant hypertension.",
-        }
-    ]
-    # The caption stays readable in the text; only the unlinkable image goes.
-    assert "Stepped care algorithm" in content
-    assert "g1f1.jpg" not in content
-
-
-def test_build_pubmed_article_text_records_supplementary_files() -> None:
-    """Evidence tables live in supplementary .docx payloads the API does not return."""
-    content, _, _, metadata = build_pubmed_article_text(
-        _article_client(_jats()),
-        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
-    )
-
-    assert metadata["supplements"] == [
-        {
-            "id": "MOESM1",
-            "file": "12345_MOESM1_ESM.docx",
-            "label": "",
-            "caption": "Additional file 1: Table S1. Evidence profile for each recommendation.",
-        }
-    ]
-    assert "12345_MOESM1_ESM.docx" not in content
-
-
-def test_build_pubmed_article_text_does_not_repeat_the_abstract_heading() -> None:
-    """JATS labels the abstract itself, which would duplicate the heading we add."""
-    content, section_count, _, _ = build_pubmed_article_text(
-        _article_client(_jats(abstract="<abstract><title>Abstract</title><p>Summary text.</p></abstract>")),
-        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
-    )
-
-    assert content.startswith("## Abstract")
-    assert "**Abstract**" not in content
-    assert "Summary text." in content
-    # Three body sections plus the abstract.
-    assert section_count == 4
-
-
-@pytest.mark.parametrize(
-    ("license_block", "expected_name", "expected_url"),
-    [
-        (_LICENSE_ALI, "CC BY", "https://creativecommons.org/licenses/by/4.0/"),
-        (_LICENSE_EXT_LINK, "CC BY-NC", "https://creativecommons.org/licenses/by-nc/4.0/"),
-        (_LICENSE_TEXT_ONLY, "CC BY-NC-ND", "https://creativecommons.org/licenses/by-nc-nd/4.0/"),
-    ],
-)
-def test_build_pubmed_article_text_reads_every_license_shape(
-    license_block: str, expected_name: str, expected_url: str
-) -> None:
-    """All three shapes occur in the open-access subset, and ND changes what we may redistribute."""
-    _, _, _, metadata = build_pubmed_article_text(
-        _article_client(_jats(license_block=license_block)),
-        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
-    )
-
-    assert metadata["license"] == expected_name
-    assert metadata["license_url"] == expected_url
-
-
-def test_build_pubmed_article_text_records_permissive_publisher_terms() -> None:
-    """A publisher license is not a missing license, so the terms are kept verbatim."""
-    _, _, _, metadata = build_pubmed_article_text(
-        _article_client(_jats(license_block=_LICENSE_NON_CC_PERMISSIVE)),
-        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
-    )
-
-    assert metadata["license"] == ""
-    assert metadata["license_url"] == ""
-    assert metadata["license_type"] == "open-access"
-    assert "unrestricted re-use" in metadata["license_statement"]
-
-
-def test_build_pubmed_article_text_records_a_reserved_copyright_statement() -> None:
-    """COVID-era free access still reserves all rights, and that text is not inside `<license>`."""
-    _, _, _, metadata = build_pubmed_article_text(
-        _article_client(_jats(license_block=_LICENSE_NON_CC_RESERVED)),
-        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
-    )
-
-    assert metadata["license"] == ""
-    # Reading only `<license>` would report free access and lose the reservation.
-    assert "free information" in metadata["license_statement"]
-    assert "All rights reserved" in metadata["copyright_statement"]
-
-
-def test_build_pubmed_article_text_rejects_a_metadata_only_record() -> None:
-    """About 2% of the subset has no deposited full text."""
-    with pytest.raises(PubMedFetchError):
-        build_pubmed_article_text(
-            _article_client(_jats(body="")),
-            PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
-        )
-
-
-def test_scrape_pubmed_article_normalizes_into_a_scraped_document() -> None:
-    """Scraped guidelines use the shared source/external_id conventions."""
-    document = scrape_pubmed_article(
-        _article_client(_jats()),
-        PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"),
-    )
-
-    assert document.source == "pubmed"
-    assert document.external_id == "pubmed-42476581"
-    assert document.url == "https://pubmed.ncbi.nlm.nih.gov/42476581/"
-    assert document.metadata["pmcid"] == "PMC13384732"
-    assert "Scope and purpose" in document.content
-
-
-def test_scrape_pubmed_pages_through_the_listing_and_terminates(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Paging advances by offset, crosses a page boundary, and stops on the empty page."""
-    seen: list[int] = []
-    handler = _listing_handler({1: ["111"], 2: ["222"]}, total=2, seen_offsets=seen)
-    monkeypatch.setattr(
-        "amfv_datasets.scraping.pubmed.default_client",
-        lambda: httpx.Client(transport=httpx.MockTransport(handler)),
-    )
-    monkeypatch.setattr(base.time, "sleep", lambda _seconds: None)
-
-    run = scrape_pubmed(documents=None)
-    documents = list(run.documents)
-
-    assert run.total == 2
-    assert [document.external_id for document in documents] == ["pubmed-111", "pubmed-222"]
-    # Page 1 is fetched once for the total and reused via `first_page_items`,
-    # so the listing is not re-requested at offset 0 during the run.
-    assert seen == [0, LISTING_PAGE_SIZE, 2 * LISTING_PAGE_SIZE]
-
-
-def test_scrape_pubmed_skips_records_pmc_holds_without_full_text(monkeypatch: pytest.MonkeyPatch) -> None:
-    """One metadata-only record does not abort a run over the rest of the listing."""
-    handler = _listing_handler({1: ["999", "222"]}, total=2)
-    monkeypatch.setattr(
-        "amfv_datasets.scraping.pubmed.default_client",
-        lambda: httpx.Client(transport=httpx.MockTransport(handler)),
-    )
-    monkeypatch.setattr(base.time, "sleep", lambda _seconds: None)
-
-    documents = list(scrape_pubmed(documents=None).documents)
-
-    assert [document.external_id for document in documents] == ["pubmed-222"]
-
-
-def test_eutils_retries_when_ncbi_rate_limits(monkeypatch: pytest.MonkeyPatch) -> None:
-    """A 429 mid-run must not kill the scrape; NCBI returns them on bursts."""
-    monkeypatch.setattr("amfv_datasets.scraping.pubmed.time.sleep", lambda _seconds: None)
-    attempts = []
-
-    def handler(request: httpx.Request) -> httpx.Response:
-        attempts.append(request.url.path)
-        if len(attempts) < 3:
-            return httpx.Response(429, text="Too Many Requests")
-        return httpx.Response(200, content=_jats().encode())
-
-    client = httpx.Client(transport=httpx.MockTransport(handler))
-    document = scrape_pubmed_article(client, PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"))
-
-    assert len(attempts) == 3
-    assert document.external_id == "pubmed-42476581"
-
-
-def test_eutils_gives_up_after_repeated_rate_limits(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Persistent 429s are a real stop signal, not something to retry forever."""
-    monkeypatch.setattr("amfv_datasets.scraping.pubmed.time.sleep", lambda _seconds: None)
-    attempts = []
-
-    def handler(request: httpx.Request) -> httpx.Response:
-        attempts.append(request.url.path)
-        return httpx.Response(429, text="Too Many Requests")
-
-    client = httpx.Client(transport=httpx.MockTransport(handler))
-    with pytest.raises(httpx.HTTPStatusError):
-        scrape_pubmed_article(client, PubMedArticleRef(pmid="42476581", pmcid="PMC13384732"))
-
-    assert len(attempts) == 4
-
-
-def test_pubmed_ref_from_url_accepts_both_pubmed_and_pmc_urls() -> None:
-    """Only one identifier is knowable from a URL; the other is resolved when scraping."""
-    pubmed_ref = pubmed_ref_from_url("https://pubmed.ncbi.nlm.nih.gov/42476581/")
-    pmc_ref = pubmed_ref_from_url("https://www.ncbi.nlm.nih.gov/pmc/articles/PMC13384732/")
-
-    assert pubmed_ref == PubMedArticleRef(pmid="42476581")
-    assert pmc_ref == PubMedArticleRef(pmcid="PMC13384732")
-    assert pmc_ref.page_url == "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC13384732/"
-
-
-def test_scrape_pubmed_article_uses_the_same_id_from_either_entry_point() -> None:
-    """A PMC URL carries no PMID, but the record does, so both routes agree."""
-    from_listing = scrape_pubmed_article(
-        _article_client(_jats()), PubMedArticleRef(pmid="42476581", pmcid="PMC13384732")
-    )
-    from_pmc_url = scrape_pubmed_article(_article_client(_jats()), PubMedArticleRef(pmcid="PMC13384732"))
-
-    assert from_listing.external_id == from_pmc_url.external_id == "pubmed-42476581"
-
-
-def test_pubmed_ref_from_url_resolves_a_pmid_to_full_text(monkeypatch: pytest.MonkeyPatch) -> None:
-    """A PubMed URL carries no PMCID, so efetch needs one looked up first."""
-    handler = _listing_handler({1: ["111"]}, total=1)
-    client = httpx.Client(transport=httpx.MockTransport(handler))
-
-    document = scrape_pubmed_article(client, pubmed_ref_from_url("https://pubmed.ncbi.nlm.nih.gov/111/"))
-
-    assert document.external_id == "pubmed-111"
-    assert document.metadata["pmcid"] == "PMC111"
-
-
-@pytest.mark.parametrize(
-    "url",
-    [
-        "https://example.org/42476581/",
-        "https://pubmed.ncbi.nlm.nih.gov/not-a-pmid/",
-        "https://www.ncbi.nlm.nih.gov/pmc/articles/13384732/",
-        "ftp://pubmed.ncbi.nlm.nih.gov/42476581/",
-    ],
-)
-def test_pubmed_ref_from_url_rejects_non_article_urls(url: str) -> None:
-    """Anything that is not a PubMed or PMC article URL is an error, not a guess."""
-    with pytest.raises(PubMedFetchError):
-        pubmed_ref_from_url(url)
diff --git a/datasets/test/test_scraping_rch.py b/datasets/test/test_scraping_rch.py
deleted file mode 100644
index 5b2c4f92..00000000
--- a/datasets/test/test_scraping_rch.py
+++ /dev/null
@@ -1,252 +0,0 @@
-"""Tests for RCH clinical guideline scraping helpers."""
-
-import httpx
-import pytest
-
-from amfv_datasets.scraping.html import LinkMode
-from amfv_datasets.scraping.rch import (
-    BASE_URL,
-    GUIDELINE_INDEX_URL,
-    RchGuidelineRef,
-    _scrape_or_skip_guideline,
-    guideline_ref_from_url,
-    list_guidelines,
-    scrape_guideline,
-)
-
-
-def test_list_guidelines_discovers_unique_html_pages() -> None:
-    """The A-Z index yields unique HTML guideline references and nested paths."""
-    html_text = """
-    <html>
-      <div id="tabnav-letter-blocks">
-        <a href="/clinicalguide/guideline_index/Acute_asthma/">Acute asthma</a>
-        <a href="/clinicalguide/guideline_index/Acute_asthma/">Asthma acute (see Acute asthma)</a>
-        <a href="/clinicalguide/guideline_index/fractures/Elbow_Dislocations/">Elbow dislocations</a>
-        <a href="/Persistent_nasal_discharge_rhinosinusitis/.aspx">Nasal discharge</a>
-        <a href="/Persistent_nasal_discharge_rhinosinusitis/.aspx">Rhinosinusitis</a>
-        <a href="/clinicalguide/guideline_index/CPG_Committee_Calendar/">Committee calendar</a>
-        <a href="/clinicalguide/guideline_index/files/package.pdf">PDF package</a>
-      </div>
-      <footer><a href="/clinicalguide/guideline_index/not-a-guideline/">Footer link</a></footer>
-    </html>
-    """
-
-    def handler(request: httpx.Request) -> httpx.Response:
-        assert str(request.url) == GUIDELINE_INDEX_URL
-        return httpx.Response(200, text=html_text)
-
-    client = httpx.Client(transport=httpx.MockTransport(handler))
-
-    assert list_guidelines(client) == [
-        RchGuidelineRef(
-            slug="Acute_asthma",
-            title="Acute asthma",
-            page_url=f"{BASE_URL}/clinicalguide/guideline_index/Acute_asthma/",
-            aliases=("Asthma acute",),
-        ),
-        RchGuidelineRef(
-            slug="fractures/Elbow_Dislocations",
-            title="Elbow dislocations",
-            page_url=f"{BASE_URL}/clinicalguide/guideline_index/fractures/Elbow_Dislocations/",
-        ),
-        RchGuidelineRef(
-            slug="Persistent_nasal_discharge_rhinosinusitis",
-            title="Nasal discharge",
-            page_url=f"{BASE_URL}/Persistent_nasal_discharge_rhinosinusitis/.aspx",
-            aliases=("Rhinosinusitis",),
-        ),
-    ]
-
-
-def test_guideline_ref_from_url_normalizes_host_and_query() -> None:
-    """RCH guideline URLs are normalized to canonical HTTPS page URLs."""
-    assert guideline_ref_from_url(
-        "http://rch.org.au/clinicalguide/guideline_index/Acute_asthma/?print=yes#management"
-    ) == RchGuidelineRef(
-        slug="Acute_asthma",
-        title="Acute asthma",
-        page_url="https://www.rch.org.au/clinicalguide/guideline_index/Acute_asthma/",
-    )
-
-
-def test_guideline_ref_from_url_preserves_legacy_aspx_route() -> None:
-    """Legacy ASPX guideline paths do not receive a breaking trailing slash."""
-    assert (
-        guideline_ref_from_url("https://www.rch.org.au/clinicalguide/guideline_index/Gastrostomy.aspx/").page_url
-        == "https://www.rch.org.au/clinicalguide/guideline_index/Gastrostomy.aspx"
-    )
-
-
-def test_guideline_ref_from_url_accepts_direct_rhinosinusitis_guideline() -> None:
-    """The clinical A-Z index's direct rhinosinusitis route remains discoverable."""
-    assert guideline_ref_from_url(
-        "https://rch.org.au/Persistent_nasal_discharge_rhinosinusitis/.aspx?print=yes"
-    ) == RchGuidelineRef(
-        slug="Persistent_nasal_discharge_rhinosinusitis",
-        title="Persistent nasal discharge rhinosinusitis",
-        page_url=f"{BASE_URL}/Persistent_nasal_discharge_rhinosinusitis/.aspx",
-    )
-
-
-def test_scrape_guideline_extracts_primary_widgets_as_markdown() -> None:
-    """Guideline content, references, metadata, lists, tables, and images are preserved."""
-    html_text = """
-    <html>
-      <h1>Acute asthma</h1>
-      <div id="rch-primary" class="col-md-10">
-        <div class="widgetBody"><img alt="PIC Endorsed" src="/pic-logo.png"></div>
-        <div class="widgetBody">
-          <h2>Key points</h2>
-          <p><strong>Treat urgently.</strong><strong></strong></p>
-          <ul>
-            <li>Assess severity:</li>
-            <ul><li>Check breathing.</li><li>Check activity.</li></ul>
-          </ul>
-          <h3><strong><strong>Management</strong> details</strong></h3>
-          <p>See the <a href="/clinicalguide/guideline_index/Anaphylaxis/">anaphylaxis guideline</a>.</p>
-          <p>Jump to <a href="#management">management</a>.</p>
-          <table><tr><th>Severity</th><th>Action</th></tr><tr><td>Severe</td><td>Escalate</td></tr></table>
-          <h4><br></h4>
-          <p><img src="/uploadedImages/asthma-flowchart.png" alt="Asthma flowchart"></p>
-          <p><img src="http://webedit.rch.org.au/uploadedImages/legacy.png" alt="Legacy image"></p>
-          <p><img alt="Missing image source"></p>
-          <p>Last updated July 2025</p>
-          <script>window.rchWidget = {"tracking": true};</script>
-        </div>
-        <div class="widgetBody"><h2>Reference List</h2><ol><li>Reference one.</li></ol></div>
-      </div>
-      <footer><h2>Contact us</h2><p>Footer content.</p></footer>
-    </html>
-    """
-    ref = RchGuidelineRef(
-        slug="Acute_asthma",
-        title="Acute asthma listing title",
-        page_url="https://www.rch.org.au/clinicalguide/guideline_index/Acute_asthma/",
-        aliases=("Asthma acute", "Wheeze"),
-    )
-
-    def handler(request: httpx.Request) -> httpx.Response:
-        assert str(request.url) == ref.page_url
-        return httpx.Response(200, text=html_text)
-
-    client = httpx.Client(transport=httpx.MockTransport(handler))
-
-    document = scrape_guideline(client, ref)
-
-    assert document.source == "rch"
-    assert document.external_id == "rch-acute-asthma"
-    assert document.title == "Acute asthma"
-    assert document.section_count == 2
-    assert document.metadata == {
-        "slug": "Acute_asthma",
-        "aliases": ["Asthma acute", "Wheeze"],
-        "last_updated": "July 2025",
-    }
-    assert "- Assess severity:\n  - Check breathing.\n  - Check activity." in document.content
-    assert "### Management details" in document.content
-    assert (
-        "[anaphylaxis guideline](https://www.rch.org.au/clinicalguide/guideline_index/Anaphylaxis/)" in document.content
-    )
-    assert (
-        "[management](https://www.rch.org.au/clinicalguide/guideline_index/Acute_asthma/#management)"
-        in document.content
-    )
-    assert "| Severity | Action |" in document.content
-    assert "![Asthma flowchart](https://www.rch.org.au/uploadedImages/asthma-flowchart.png)" in document.content
-    assert "![Legacy image](https://www.rch.org.au/uploadedImages/legacy.png)" in document.content
-    assert "Missing image source" not in document.content
-    assert "## Reference List" in document.content
-    assert "****" not in document.content
-    assert "PIC Endorsed" not in document.content
-    assert "Footer content" not in document.content
-    assert "rchWidget" not in document.content
-
-
-def test_scrape_guideline_keeps_substantial_headingless_legacy_content() -> None:
-    """Legacy clinical tables without section headings remain valid guideline content."""
-    clinical_rows = "".join(f"<tr><td>Disease {index}</td><td>Dose {index}</td></tr>" for index in range(10))
-    html_text = f"""
-    <html><h1>Empiric treatment</h1><div id="rch-primary"><div class="widgetBody">
-      <p>Start empiric treatment based on clinical features and local antimicrobial susceptibility patterns.</p>
-      <table><tr><th>Disease</th><th>Treatment</th></tr>{clinical_rows}</table>
-    </div></div></html>
-    """
-    ref = RchGuidelineRef("Empiric_treatment", "Empiric treatment", f"{BASE_URL}/guideline/empiric")
-
-    with httpx.Client(transport=httpx.MockTransport(lambda _request: httpx.Response(200, text=html_text))) as client:
-        document = scrape_guideline(client, ref)
-
-    assert "| Disease | Treatment |" in document.content
-    assert "local antimicrobial susceptibility patterns" in document.content
-
-
-def test_scrape_guideline_resolves_same_site_link_wrapper() -> None:
-    """Link-only A-Z entries resolve to their substantial same-site content pages."""
-    index_url = f"{BASE_URL}/clinicalguide/guideline_index/IV_Immunoglobulin/"
-    target_url = f"{BASE_URL}/bloodtrans/about_blood_products/Intravenous_Immunoglobulin_Guideline"
-    wrapper_html = f"""
-    <html><h1>IV Immunoglobulin</h1><div id="rch-primary"><div class="widgetBody">
-      <p><a href="{target_url}">Go to the IVIg guideline</a></p>
-    </div></div></html>
-    """
-    target_html = """
-    <html><h1>Intravenous immunoglobulin</h1><div id="rch-primary"><div class="widgetBody">
-      <h2>Administration</h2><p>Monitor the patient throughout the infusion.</p>
-    </div></div></html>
-    """
-    ref = RchGuidelineRef("IV_Immunoglobulin", "IV Immunoglobulin", index_url)
-
-    def handler(request: httpx.Request) -> httpx.Response:
-        pages = {index_url: wrapper_html, target_url: target_html}
-        return httpx.Response(200, text=pages[str(request.url)])
-
-    with httpx.Client(transport=httpx.MockTransport(handler)) as client:
-        document = scrape_guideline(client, ref)
-
-    assert document.url == target_url
-    assert document.title == "Intravenous immunoglobulin"
-    assert document.metadata == {"slug": "IV_Immunoglobulin", "index_url": index_url}
-    assert "Monitor the patient throughout the infusion." in document.content
-
-
-def test_scrape_or_skip_guideline_logs_non_guideline_index_entry(caplog: pytest.LogCaptureFixture) -> None:
-    """Resource pages discovered in the guideline index are reported and skipped."""
-    ref = RchGuidelineRef(
-        slug="Immigrant_health_resources",
-        title="Immigrant health resources",
-        page_url="https://www.rch.org.au/clinicalguide/guideline_index/Immigrant_health_resources/",
-    )
-
-    def handler(request: httpx.Request) -> httpx.Response:
-        assert str(request.url) == ref.page_url
-        html_text = "<html><div id='rch-primary'><div class='widgetBody'>Resource link</div></div></html>"
-        return httpx.Response(200, text=html_text)
-
-    with httpx.Client(transport=httpx.MockTransport(handler)) as client:
-        assert _scrape_or_skip_guideline(client, ref, link_mode=LinkMode.KEEP) is None
-
-    assert "Skipping unsupported RCH index entry" in caplog.text
-
-
-@pytest.mark.parametrize("status_code", [404, 410])
-def test_scrape_or_skip_guideline_logs_stale_index_entry(
-    status_code: int,
-    caplog: pytest.LogCaptureFixture,
-) -> None:
-    """Permanently missing pages in the live index are reported and skipped."""
-    ref = RchGuidelineRef("stale", "Stale", f"{BASE_URL}/clinicalguide/guideline_index/stale/")
-
-    with httpx.Client(transport=httpx.MockTransport(lambda _request: httpx.Response(status_code))) as client:
-        assert _scrape_or_skip_guideline(client, ref, link_mode=LinkMode.KEEP) is None
-
-    assert f"HTTP {status_code}" in caplog.text
-
-
-def test_scrape_or_skip_guideline_raises_transient_http_error() -> None:
-    """Server errors still abort so incomplete runs are not silently accepted."""
-    ref = RchGuidelineRef("error", "Error", f"{BASE_URL}/clinicalguide/guideline_index/error/")
-
-    with httpx.Client(transport=httpx.MockTransport(lambda _request: httpx.Response(503))) as client:
-        with pytest.raises(httpx.HTTPStatusError):
-            _scrape_or_skip_guideline(client, ref, link_mode=LinkMode.KEEP)
diff --git a/datasets/test/test_scraping_who.py b/datasets/test/test_scraping_who.py
deleted file mode 100644
index 110c5a56..00000000
--- a/datasets/test/test_scraping_who.py
+++ /dev/null
@@ -1,154 +0,0 @@
-"""Tests for WHO scraping helpers."""
-
-import json
-from pathlib import Path
-
-import httpx
-import pytest
-
-from amfv_datasets.scraping.html import LinkMode
-from amfv_datasets.scraping.who import (
-    BASE_URL,
-    WHO_LICENSE,
-    WhoFetchError,
-    WhoListingPage,
-    WhoPublicationRef,
-    build_publication_text,
-    list_publications,
-    publication_ref_from_url,
-    scrape_publication,
-)
-
-_FIXTURES = Path(__file__).parent / "fixtures"
-_CVC_GUIDELINE_TITLE = (
-    "Guidelines for the prevention of bloodstream infections and other infections "
-    "associated with the use of intravascular catheters: part 2: central venous catheters"
-)
-_CERVICAL_GUIDELINE_TITLE = (
-    "WHO guideline for screening and treatment of cervical pre-cancer lesions for cervical cancer prevention"
-)
-
-
-def test_list_publications_parses_api_listing() -> None:
-    """WHO listing payloads are parsed from the publications OData API."""
-    payload = json.loads((_FIXTURES / "who_listing_api.json").read_text(encoding="utf-8"))
-
-    def handler(request: httpx.Request) -> httpx.Response:
-        assert request.url.path == "/api/hubs/publications"
-        assert request.url.params["$filter"] == "publishingoffices/any(s:s eq c09761c0-ab8e-4cfa-9744-99509c4d306b)"
-        assert request.url.params["$skip"] == "0"
-        return httpx.Response(200, json=payload)
-
-    client = httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL)
-
-    listing_page = list_publications(client)
-
-    assert listing_page == WhoListingPage(
-        total=356,
-        refs=[
-            WhoPublicationRef(
-                publication_id="9789240121805",
-                title=_CVC_GUIDELINE_TITLE,
-                page_url="https://www.who.int/publications/i/item/9789240121805",
-                publication_date="28 May 2026",
-                tag="Guideline",
-                download_url="https://iris.who.int/server/api/core/bitstreams/f750f24d-c0c2-425c-85fd-ec310d2ce994/content",
-            ),
-            WhoPublicationRef(
-                publication_id="9789240121744",
-                title=_CERVICAL_GUIDELINE_TITLE,
-                page_url="https://www.who.int/publications/i/item/9789240121744",
-                publication_date="8 May 2026",
-                tag="Guideline",
-                download_url="https://iris.who.int/server/api/core/bitstreams/32214b73-0e95-4243-9e83-617948510dcd/content",
-            ),
-        ],
-    )
-
-
-def test_publication_ref_from_url_normalizes_publication_url() -> None:
-    """WHO publication URLs are normalized to canonical refs."""
-    assert publication_ref_from_url("https://www.who.int/publications/i/item/9789240121805") == WhoPublicationRef(
-        publication_id="9789240121805",
-        title="9789240121805",
-        page_url="https://www.who.int/publications/i/item/9789240121805",
-    )
-
-
-def test_build_publication_text_scrapes_overview() -> None:
-    """Publication pages are converted into Overview markdown."""
-    html = (_FIXTURES / "who_publication_overview.html").read_text(encoding="utf-8")
-    page_url = "https://www.who.int/publications/i/item/9789240121805"
-
-    def handler(request: httpx.Request) -> httpx.Response:
-        assert str(request.url) == page_url
-        return httpx.Response(200, text=html)
-
-    client = httpx.Client(transport=httpx.MockTransport(handler))
-    ref = WhoPublicationRef(
-        publication_id="9789240121805",
-        title=_CVC_GUIDELINE_TITLE,
-        page_url=page_url,
-        publication_date="28 May 2026",
-        tag="Guideline",
-        download_url="https://iris.who.int/server/api/core/bitstreams/f750f24d-c0c2-425c-85fd-ec310d2ce994/content",
-    )
-
-    content, section_count, title, metadata = build_publication_text(client, ref)
-
-    assert title.startswith("Guidelines for the prevention of bloodstream infections")
-    assert section_count == 1
-    assert "### Overview" in content
-    assert "central venous catheters (CVCs)" in content
-    assert "WHO Team" not in content
-    assert metadata["publication_date"] == "28 May 2026"
-    assert metadata["tag"] == "Guideline"
-    assert metadata["isbn"] == "978-92-4-012180-5"
-    assert metadata["content_scope"] == "overview"
-    assert metadata["license"] == WHO_LICENSE
-    assert metadata["listing_category"] == "who-guidelines"
-
-
-def test_scrape_publication_builds_scraped_document() -> None:
-    """A WHO publication ref is normalized into a ScrapedDocument."""
-    html = (_FIXTURES / "who_publication_overview.html").read_text(encoding="utf-8")
-    page_url = "https://www.who.int/publications/i/item/9789240121805"
-
-    def handler(request: httpx.Request) -> httpx.Response:
-        return httpx.Response(200, text=html)
-
-    client = httpx.Client(transport=httpx.MockTransport(handler))
-    ref = WhoPublicationRef(
-        publication_id="9789240121805",
-        title=_CVC_GUIDELINE_TITLE,
-        page_url=page_url,
-    )
-
-    document = scrape_publication(client, ref, link_mode=LinkMode.STRIP)
-
-    assert document.source == "who"
-    assert document.external_id == "who-9789240121805"
-    assert document.url == page_url
-    assert document.section_count == 1
-    assert document.content.strip()
-    assert document.metadata["publication_id"] == "9789240121805"
-    assert document.metadata["content_scope"] == "overview"
-
-
-def test_build_publication_text_raises_when_overview_missing() -> None:
-    """Missing publication markup raises WhoFetchError."""
-    html = "<html><body><p>No publication section here.</p></body></html>"
-    page_url = "https://www.who.int/publications/i/item/9789240121805"
-
-    def handler(request: httpx.Request) -> httpx.Response:
-        return httpx.Response(200, text=html)
-
-    client = httpx.Client(transport=httpx.MockTransport(handler))
-    ref = WhoPublicationRef(
-        publication_id="9789240121805",
-        title="Example guideline",
-        page_url=page_url,
-    )
-
-    with pytest.raises(WhoFetchError, match="No publication content section"):
-        build_publication_text(client, ref)

From c07e4119a6623eea6fe8161675fe9632cafe2c4e Mon Sep 17 00:00:00 2001
From: Zander Giuffrida <zndr27@gmail.com>
Date: Mon, 17 Aug 2026 23:37:52 -0700
Subject: [PATCH 13/15] dispatch scrapers from one registry

Registering a source was an enum member, a match arm, an _expand_source entry
and an __init__ re-export block, in two files. It is now one entry in SCRAPERS.

Also fixes --source all, which returned inside the loop and so only ever ran
the first source.
---
 datasets/amfv_datasets/scraping/__init__.py | 32 ++++--------
 datasets/amfv_datasets/scraping/cli.py      | 54 +++++++++++++--------
 datasets/test/test_scraping_cli.py          | 34 ++++++++++---
 3 files changed, 69 insertions(+), 51 deletions(-)

diff --git a/datasets/amfv_datasets/scraping/__init__.py b/datasets/amfv_datasets/scraping/__init__.py
index cd7cdb06..4ed0d214 100644
--- a/datasets/amfv_datasets/scraping/__init__.py
+++ b/datasets/amfv_datasets/scraping/__init__.py
@@ -1,4 +1,8 @@
-"""Web scraping helpers and source-specific scrapers."""
+"""Web scraping helpers and source-specific scrapers.
+
+Only the shared scraping contract is re-exported here. Import a source's own
+symbols from its module, so registering a source touches one file.
+"""
 
 from amfv_datasets.scraping.base import (
     USER_AGENT,
@@ -8,7 +12,7 @@
     default_client,
     scrape_listing_documents,
 )
-from amfv_datasets.scraping.cli import OutputFormat, ScraperSource
+from amfv_datasets.scraping.cli import ALL_SOURCES, SCRAPERS, OutputFormat, Scraper
 from amfv_datasets.scraping.html import (
     LinkMode,
     absolute_unique_urls,
@@ -17,38 +21,22 @@
     first_matching_urls,
     html_to_markdown,
 )
-from amfv_datasets.scraping.nice import (
-    GuidanceListingPage,
-    GuidanceRef,
-    NiceFetchError,
-    build_guideline_text,
-    guidance_ref_from_url,
-    list_published_guidance,
-    scrape_guideline,
-    scrape_nice,
-)
 
 __all__ = [
-    "GuidanceRef",
-    "GuidanceListingPage",
+    "ALL_SOURCES",
     "LinkMode",
-    "NiceFetchError",
     "OutputFormat",
+    "SCRAPERS",
     "ScrapeError",
     "ScrapeRun",
     "ScrapedDocument",
-    "ScraperSource",
+    "Scraper",
     "USER_AGENT",
     "absolute_unique_urls",
-    "build_guideline_text",
     "clean_text",
-    "document_title",
     "default_client",
+    "document_title",
     "first_matching_urls",
-    "guidance_ref_from_url",
     "html_to_markdown",
-    "list_published_guidance",
-    "scrape_guideline",
     "scrape_listing_documents",
-    "scrape_nice",
 ]
diff --git a/datasets/amfv_datasets/scraping/cli.py b/datasets/amfv_datasets/scraping/cli.py
index e8aece34..acdcd050 100644
--- a/datasets/amfv_datasets/scraping/cli.py
+++ b/datasets/amfv_datasets/scraping/cli.py
@@ -8,8 +8,9 @@
 from collections.abc import Iterable
 from dataclasses import asdict
 from enum import StrEnum
+from itertools import chain
 from pathlib import Path
-from typing import Annotated, TextIO
+from typing import Annotated, Protocol, TextIO
 
 import typer
 from rich.console import Console
@@ -29,11 +30,20 @@
 from amfv_datasets.scraping.nice import scrape_nice
 
 
-class ScraperSource(StrEnum):
-    """Supported scraper sources."""
+class Scraper(Protocol):
+    """Entry point a source module exposes to run one scrape."""
 
-    ALL = "all"
-    NICE = "nice"
+    def __call__(self, *, documents: int | None, link_mode: LinkMode, url: str | None) -> ScrapeRun:
+        """Scrape a source into a run of documents."""
+        ...
+
+
+ALL_SOURCES = "all"
+
+SCRAPERS: dict[str, Scraper] = {
+    "nice": scrape_nice,
+}
+"""Scraper entry point by source name. Adding a source is an import and an entry here."""
 
 
 class OutputFormat(StrEnum):
@@ -48,7 +58,7 @@ class OutputFormat(StrEnum):
 
 
 def scrape_documents(
-    source: ScraperSource,
+    source: str,
     *,
     documents: int | None,
     link_mode: LinkMode,
@@ -57,8 +67,8 @@ def scrape_documents(
     """Configure a scrape for a source.
 
     Args:
-        source: Scraper source to run. Use `ScraperSource.ALL` to run every
-            implemented source.
+        source: Scraper source to run. Use `ALL_SOURCES` to run every registered
+            source.
         documents: Number of documents to scrape. When unset, each source runs
             until it is exhausted (default: None).
         link_mode: Whether links are kept as markdown links or stripped to their
@@ -68,13 +78,11 @@ def scrape_documents(
     if documents is not None and documents < 1:
         raise ValueError(f"documents must be at least 1; got {documents}")
 
-    for selected_source in _expand_source(source):
-        match selected_source:
-            case ScraperSource.NICE:
-                return scrape_nice(documents=documents, link_mode=link_mode, url=url)
-            case ScraperSource.ALL:
-                raise AssertionError("expanded source cannot be all")
-    raise AssertionError(f"unsupported source: {source}")
+    scrape_runs = [SCRAPERS[name](documents=documents, link_mode=link_mode, url=url) for name in _expand_source(source)]
+    if len(scrape_runs) == 1:
+        return scrape_runs[0]
+    total = None if any(run.total is None for run in scrape_runs) else sum(run.total or 0 for run in scrape_runs)
+    return ScrapeRun(documents=chain.from_iterable(run.documents for run in scrape_runs), total=total)
 
 
 def write_jsonl(documents: Iterable[ScrapedDocument], output: TextIO) -> int:
@@ -123,15 +131,17 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path
     return count
 
 
-def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]:
-    if source is ScraperSource.ALL:
-        return (ScraperSource.NICE,)
+def _expand_source(source: str) -> tuple[str, ...]:
+    if source == ALL_SOURCES:
+        return tuple(SCRAPERS)
+    if source not in SCRAPERS:
+        raise typer.BadParameter(f"unknown source {source!r}; choose from {', '.join([ALL_SOURCES, *SCRAPERS])}")
     return (source,)
 
 
 @app.command(help="Run a scraper and write the scraped documents.")
 def run(
-    source: Annotated[ScraperSource, typer.Option("--source", help="Scraper source to run.")],
+    source: Annotated[str, typer.Option("--source", help=f"Scraper source to run: {', '.join([ALL_SOURCES, *SCRAPERS])}.")],  # noqa: E501
     url: Annotated[str | None, typer.Option("--url", help="Source URL to scrape as a single document.")] = None,
     documents: Annotated[str, typer.Option("--documents", help="Number of documents to scrape, or 'all'.")] = "1",
     link_mode: Annotated[LinkMode, typer.Option("--links", help="Whether to keep markdown links or strip links to text.")] = LinkMode.KEEP,  # noqa: E501
@@ -168,7 +178,7 @@ def run(
         if output_path is None:
             raise typer.BadParameter("--output is required when --format markdown")
         count = write_markdown_files(scraped_documents, output_path)
-    target = url or source.value
+    target = url or source
     typer.echo(f"scraped {count} documents from {target}", err=True)
 
 
@@ -248,8 +258,10 @@ def main() -> None:
 
 
 __all__ = [
+    "ALL_SOURCES",
+    "SCRAPERS",
     "OutputFormat",
-    "ScraperSource",
+    "Scraper",
     "app",
     "LinkMode",
     "main",
diff --git a/datasets/test/test_scraping_cli.py b/datasets/test/test_scraping_cli.py
index 44f714fa..bb24a7f6 100644
--- a/datasets/test/test_scraping_cli.py
+++ b/datasets/test/test_scraping_cli.py
@@ -9,7 +9,9 @@
 
 from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun
 from amfv_datasets.scraping.cli import (
-    ScraperSource,
+    ALL_SOURCES,
+    SCRAPERS,
+    _expand_source,
     app,
     write_huggingface_dataset,
     write_jsonl,
@@ -65,13 +67,13 @@ def test_cli_run_writes_jsonl_to_stdout(monkeypatch: pytest.MonkeyPatch) -> None
     runner = CliRunner()
 
     def fake_scrape_documents(
-        source: ScraperSource,
+        source: str,
         *,
         documents: int | None,
         link_mode: LinkMode,
         url: str | None = None,
     ) -> ScrapeRun:
-        assert source is ScraperSource.NICE
+        assert source == "nice"
         assert documents == 3
         assert link_mode is LinkMode.STRIP
         assert url is None
@@ -100,7 +102,7 @@ def test_cli_run_can_disable_progress_for_file_output(monkeypatch: pytest.Monkey
     runner = CliRunner()
 
     def fake_scrape_documents(
-        source: ScraperSource,
+        source: str,
         *,
         documents: int | None,
         link_mode: LinkMode,
@@ -132,7 +134,7 @@ def test_cli_run_uses_progress_by_default(monkeypatch: pytest.MonkeyPatch, tmp_p
     progress_calls = 0
 
     def fake_scrape_documents(
-        source: ScraperSource,
+        source: str,
         *,
         documents: int | None,
         link_mode: LinkMode,
@@ -182,7 +184,7 @@ def fake_progress(
         assert total == 1
         yield from documents
 
-    monkeypatch.setattr("amfv_datasets.scraping.cli.scrape_nice", fake_scrape_nice)
+    monkeypatch.setitem(SCRAPERS, "nice", fake_scrape_nice)
     monkeypatch.setattr("amfv_datasets.scraping.cli._progress_documents", fake_progress)
 
     result = runner.invoke(
@@ -199,13 +201,13 @@ def test_cli_run_accepts_all_documents(monkeypatch: pytest.MonkeyPatch) -> None:
     runner = CliRunner()
 
     def fake_scrape_documents(
-        source: ScraperSource,
+        source: str,
         *,
         documents: int | None,
         link_mode: LinkMode,
         url: str | None = None,
     ) -> ScrapeRun:
-        assert source is ScraperSource.ALL
+        assert source == "all"
         assert documents is None
         assert link_mode is LinkMode.KEEP
         assert url is None
@@ -219,6 +221,22 @@ def fake_scrape_documents(
     assert json.loads(result.stdout.splitlines()[0])["external_id"] == "nice-ng1"
 
 
+def test_cli_run_rejects_an_unregistered_source() -> None:
+    """An unknown --source names the sources that are registered."""
+    runner = CliRunner()
+
+    result = runner.invoke(app, ["--source", "nhs"])
+
+    assert result.exit_code != 0
+    assert "'nhs'" in result.stderr
+    assert "all, nice" in result.stderr
+
+
+def test_expand_source_runs_every_registered_scraper() -> None:
+    """The all source expands to the registry rather than a hand-written list."""
+    assert _expand_source(ALL_SOURCES) == tuple(SCRAPERS)
+
+
 def _document() -> ScrapedDocument:
     return ScrapedDocument(
         source="nice",

From 4f90b9288659ac7e62382ee269f48702dc4696f3 Mon Sep 17 00:00:00 2001
From: Zander Giuffrida <zndr27@gmail.com>
Date: Mon, 17 Aug 2026 23:38:26 -0700
Subject: [PATCH 14/15] generalize the client type in scrape_listing_documents

A source whose client is not httpx (Playwright, for one) can drive the same
loop. The loop passes the client through untouched, so this is types only.
---
 datasets/amfv_datasets/scraping/base.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/datasets/amfv_datasets/scraping/base.py b/datasets/amfv_datasets/scraping/base.py
index 3651cdb7..9c278e08 100644
--- a/datasets/amfv_datasets/scraping/base.py
+++ b/datasets/amfv_datasets/scraping/base.py
@@ -69,12 +69,12 @@ def default_client(
     return httpx.Client(headers=client_headers, timeout=timeout, follow_redirects=follow_redirects)
 
 
-def scrape_listing_documents[ListingItemT](
+def scrape_listing_documents[ClientT, ListingItemT](
     *,
     documents: int | None,
-    client_factory: Callable[[], AbstractContextManager[httpx.Client]],
-    list_page: Callable[[httpx.Client, int], Iterable[ListingItemT]],
-    scrape_item: Callable[[httpx.Client, ListingItemT], ScrapedDocument | None],
+    client_factory: Callable[[], AbstractContextManager[ClientT]],
+    list_page: Callable[[ClientT, int], Iterable[ListingItemT]],
+    scrape_item: Callable[[ClientT, ListingItemT], ScrapedDocument | None],
     document_delay_seconds: float = 5.0,
     first_page_items: Iterable[ListingItemT] | None = None,
 ) -> Iterable[ScrapedDocument]:
@@ -83,7 +83,8 @@ def scrape_listing_documents[ListingItemT](
     Args:
         documents: Number of documents to scrape. When unset, listing pages are
             fetched until a page returns no items (default: None).
-        client_factory: Factory returning a context-managed HTTP client.
+        client_factory: Factory returning a context-managed client, passed to
+            `list_page` and `scrape_item` unchanged.
         list_page: Function that lists source-specific items for a page.
         scrape_item: Function that scrapes one listed item into a document. It
             may return None to skip a discovered item that is not a document.

From 0f85dd81e0d5c1cf507edafd5bafa7871bbc4776 Mon Sep 17 00:00:00 2001
From: Zander Giuffrida <zndr27@gmail.com>
Date: Mon, 17 Aug 2026 23:38:53 -0700
Subject: [PATCH 15/15] leave in-page anchors alone when absolutizing links

urljoin turned '#recommendations' into the page URL, so a link to a heading in
the same document pointed at the document instead.
---
 datasets/amfv_datasets/scraping/html.py | 3 ++-
 datasets/test/test_scraping_html.py     | 9 +++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/datasets/amfv_datasets/scraping/html.py b/datasets/amfv_datasets/scraping/html.py
index 15805281..2e670d0a 100644
--- a/datasets/amfv_datasets/scraping/html.py
+++ b/datasets/amfv_datasets/scraping/html.py
@@ -118,7 +118,8 @@ def html_to_markdown(
 def _absolutize_links(html_text: str, *, base_url: str) -> str:
     root = lxml_html.fragment_fromstring(html_text, create_parent="div")
     for link in root.xpath(".//a[@href]"):
-        link.set("href", urljoin(base_url, link.get("href")))
+        href = link.get("href")
+        link.set("href", href if href.startswith("#") else urljoin(base_url, href))
     for image in root.xpath(".//img[@src]"):
         image.set("src", urljoin(base_url, image.get("src")))
     return "".join(lxml_html.tostring(child, encoding="unicode") for child in root)
diff --git a/datasets/test/test_scraping_html.py b/datasets/test/test_scraping_html.py
index b228222d..69a0ffe2 100644
--- a/datasets/test/test_scraping_html.py
+++ b/datasets/test/test_scraping_html.py
@@ -77,3 +77,12 @@ def test_html_to_markdown_absolutizes_images() -> None:
     assert html_to_markdown(html_text, base_url="https://example.org/guideline/") == (
         "![Treatment flowchart](https://example.org/images/flowchart.png)"
     )
+
+
+def test_html_to_markdown_keeps_in_page_anchors_relative() -> None:
+    """A link to a heading in the same document is not rewritten to the base URL."""
+    html_text = '<p>See <a href="#recommendations">recommendations</a>.</p>'
+
+    assert html_to_markdown(html_text, base_url="https://example.org/guideline/") == (
+        "See [recommendations](#recommendations)."
+    )