diff --git a/src/pyrefdev/indexer/crawl_docs.py b/src/pyrefdev/indexer/crawl_docs.py index 283f9d4..f21be5c 100644 --- a/src/pyrefdev/indexer/crawl_docs.py +++ b/src/pyrefdev/indexer/crawl_docs.py @@ -209,6 +209,12 @@ def crawl_package(pkg: Package): console.warning( f"{pkg.pypi}'s latest version {package_version!s} is older than previously crawled {crawl_state.package_version}" ) + if crawl_state is not None and crawl_state.recorded_nothing(): + console.warning( + f"{pkg.pypi}'s crawl state recorded no page, error or " + f"redirect, re-crawling from {pkg.index_url}" + ) + crawl_state = None else: crawl_state = None crawler = _Crawler( diff --git a/src/pyrefdev/indexer/index.py b/src/pyrefdev/indexer/index.py index 9fe3aba..7e8f727 100644 --- a/src/pyrefdev/indexer/index.py +++ b/src/pyrefdev/indexer/index.py @@ -113,6 +113,16 @@ class IndexState: # resumed rather than mistaken for a finished crawl. completed: bool = False + def recorded_nothing(self) -> bool: + """Whether no URL in this state ever reached a terminal outcome. + + Saving a page, failing to fetch one, and following a redirect out of the + docs all leave a trace, so a state with none of them never observed the + site at all. Resuming one finds an empty frontier and would declare the + crawl finished without making a single request. + """ + return not (self.file_to_urls or self.failed_urls or self.redirects) + @classmethod def loads(cls, content: str) -> "IndexState": return cls(**json.loads(content)) diff --git a/src/pyrefdev/indexer/parse_docs.py b/src/pyrefdev/indexer/parse_docs.py index a810310..6f6e55c 100644 --- a/src/pyrefdev/indexer/parse_docs.py +++ b/src/pyrefdev/indexer/parse_docs.py @@ -187,6 +187,13 @@ def parse(file, url): lines.extend(_create_symbols_map(symbol_to_urls)) mapping_file = Path(mapping.__file__).parent / f"{package.pypi}.py" if in_place: + if _would_erase_mapping(package, symbol_to_urls): + console.warning( + f"Not writing {mapping_file.name}: the crawled docs yielded no " + f"symbol beyond the seeds, which would erase the existing mapping. " + f"Re-crawl {package.pypi} to rebuild it." + ) + return mapping_file.write_text("\n".join(itertools.chain(lines, [""]))) else: if mapping_file.exists(): @@ -205,6 +212,20 @@ def parse(file, url): console.print("".join(diffs)) +def _would_erase_mapping(package: Package, symbol_to_urls: dict[str, str]) -> bool: + """Whether writing this mapping would drop symbols that are already shipped. + + Parsing zero crawled files still produces the seed entries, so an empty parse + is indistinguishable from a package that genuinely documents nothing unless + the mapping already on disk is consulted. + """ + seeds = _SPECIAL_SYMBOLS if package.is_cpython() else set(package.namespaces) + if set(symbol_to_urls) - set(seeds): + return False + previous = mapping.PACKAGE_INFO_MAPPING.get(package.pypi) + return previous is not None and len(previous.mapping) > len(symbol_to_urls) + + def _read_doc(package_docs: Path, file: str) -> str: filepath = package_docs / file if not filepath.exists():