crawler fix: never treat a crawl state that recorded nothing as a finished crawl.

src/pyrefdev/indexer/crawl_docs.py

@@ -209,6 +209,12 @@ def crawl_package(pkg: Package):
                         console.warning(
                             f"{pkg.pypi}'s latest version {package_version!s} is older than previously crawled {crawl_state.package_version}"
                         )
+                    if crawl_state is not None and crawl_state.recorded_nothing():
+                        console.warning(
+                            f"{pkg.pypi}'s crawl state recorded no page, error or "
+                            f"redirect, re-crawling from {pkg.index_url}"
+                        )
+                        crawl_state = None
                 else:
                     crawl_state = None
                 crawler = _Crawler(

src/pyrefdev/indexer/index.py

@@ -113,6 +113,16 @@ class IndexState:
     # resumed rather than mistaken for a finished crawl.
     completed: bool = False
 
+    def recorded_nothing(self) -> bool:
+        """Whether no URL in this state ever reached a terminal outcome.
+
+        Saving a page, failing to fetch one, and following a redirect out of the
+        docs all leave a trace, so a state with none of them never observed the
+        site at all. Resuming one finds an empty frontier and would declare the
+        crawl finished without making a single request.
+        """
+        return not (self.file_to_urls or self.failed_urls or self.redirects)
+
     @classmethod
     def loads(cls, content: str) -> "IndexState":
         return cls(**json.loads(content))

src/pyrefdev/indexer/parse_docs.py

@@ -187,6 +187,13 @@ def parse(file, url):
     lines.extend(_create_symbols_map(symbol_to_urls))
     mapping_file = Path(mapping.__file__).parent / f"{package.pypi}.py"
     if in_place:
+        if _would_erase_mapping(package, symbol_to_urls):
+            console.warning(
+                f"Not writing {mapping_file.name}: the crawled docs yielded no "
+                f"symbol beyond the seeds, which would erase the existing mapping. "
+                f"Re-crawl {package.pypi} to rebuild it."
+            )
+            return
         mapping_file.write_text("\n".join(itertools.chain(lines, [""])))
     else:
         if mapping_file.exists():
@@ -205,6 +212,20 @@ def parse(file, url):
             console.print("".join(diffs))
 
 
+def _would_erase_mapping(package: Package, symbol_to_urls: dict[str, str]) -> bool:
+    """Whether writing this mapping would drop symbols that are already shipped.
+
+    Parsing zero crawled files still produces the seed entries, so an empty parse
+    is indistinguishable from a package that genuinely documents nothing unless
+    the mapping already on disk is consulted.
+    """
+    seeds = _SPECIAL_SYMBOLS if package.is_cpython() else set(package.namespaces)
+    if set(symbol_to_urls) - set(seeds):
+        return False
+    previous = mapping.PACKAGE_INFO_MAPPING.get(package.pypi)
+    return previous is not None and len(previous.mapping) > len(symbol_to_urls)
+
+
 def _read_doc(package_docs: Path, file: str) -> str:
     filepath = package_docs / file
     if not filepath.exists():