gh-122931 CI for Linux multiarch co-installability #152831
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| CI Tests to ensure Debian `multi-arch co-installability | ||
| <https://www.debian.org/doc/debian-policy/ch-controlfields.html#multi-arch>`_ | ||
| of Python. |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| #!/usr/bin/env python3 | ||
| # Compare that multiple installs of Python don't have conflicting files | ||
| # | ||
| # This is a requirement for Debian's Multi-Arch installs of Python | ||
| # https://www.debian.org/doc/debian-policy/ch-controlfields.html#multi-arch | ||
|
|
||
| from argparse import ArgumentParser | ||
| from typing import Any | ||
| from pathlib import Path | ||
| import gzip | ||
| import json | ||
|
|
||
|
|
||
| def compare_trees(base: Path) -> bool: | ||
| """Compare all json manifests inside the directory at base.""" | ||
| hashes_seen: dict[str, str] = {} | ||
| tags_seen_by_platform: dict[str, set[frozenset[str]]] = {} | ||
|
|
||
| success: bool = True | ||
| for tree in base.iterdir(): | ||
| if not tree.is_file(): | ||
| continue | ||
|
|
||
| hashes: dict[str, str] = {} | ||
| print(f"Examining {tree}") | ||
| with gzip.open(tree) as f: | ||
| data = json.load(f) | ||
| build_details = data["build_details"] | ||
| hashes = data["hashes"] | ||
| tags_seen_by_platform.setdefault(build_details["platform"], set()).add(frozenset(build_details["abi"]["flags"])) | ||
|
|
||
| for path, digest in hashes.items(): | ||
| if is_ignored(path, build_details): | ||
| continue | ||
| if path not in hashes_seen: | ||
| hashes_seen[path] = digest | ||
| continue | ||
| if digest != hashes_seen[path]: | ||
| print(f"Mismatch found in {tree}: {path}") | ||
| print(f"{digest} != {hashes_seen[path]}") | ||
| success = False | ||
|
|
||
| # Did we see enough builds to make a useful comparison? | ||
| if len(tags_seen_by_platform) < 2: | ||
| print( | ||
| "Insufficient platforms (architectures) to compare. Expected >= 2" | ||
| ) | ||
| success = False | ||
|
|
||
| for platform, tagsets in tags_seen_by_platform.items(): | ||
| if len(tagsets) >= 2: | ||
| break | ||
| else: | ||
| print("Insufficient configuration variants tested. Expected >= 2") | ||
| success = False | ||
|
|
||
| return success | ||
|
|
||
|
|
||
| def is_ignored(pathname: str, build_details: dict[str, Any]) -> bool: | ||
| """Is this a path that we should ignore?""" | ||
|
|
||
| path = Path(pathname) | ||
|
|
||
| if path.parent.name == "__pycache__": | ||
| # Includes a timestamp, we expect a mismatch | ||
| return True | ||
|
|
||
| if path.is_relative_to("usr/bin"): | ||
| # Only libraries are multi-arch co-installed, only one arch can | ||
| # have binaries in /usr/bin at a time. | ||
| return True | ||
|
|
||
| in_usr_include = path.is_relative_to("usr/include") | ||
| if in_usr_include and path.name == "pyconfig.h": | ||
| # Varies according to config, installed into a tag-specific | ||
| # include directory | ||
| return True | ||
|
|
||
| in_usr_lib = path.is_relative_to("usr/lib") | ||
| in_pkgconfig = in_usr_lib and path.parent.name == "pkgconfig" | ||
| if in_pkgconfig and path.name in ("python3.pc", "python3-embed.pc"): | ||
| # Only the tag-suffixed .pc files are co-installable | ||
| return True | ||
|
|
||
| version = build_details["language"]["version"] | ||
| if ( | ||
| in_pkgconfig | ||
| and build_details["abi"]["flags"] # non-default install | ||
| and path.name in (f"python-{version}.pc", f"python-{version}-embed.pc") | ||
| ): | ||
| # Only the tag-suffixed .pc files are co-installable | ||
| return True | ||
|
|
||
| in_dist_info = path.parent.name.endswith(".dist-info") | ||
| if in_dist_info and path.name in ("RECORD", "WHEEL"): | ||
| # RECORD: Contains hashes, not co-installable. | ||
| # WHEEL: Contains arch and version tags. Tags can be merged but | ||
| # not architectures. | ||
| return True | ||
|
|
||
| in_site_packages = path.parent.name == "site-packages" | ||
| if in_site_packages and path.name.endswith((".abi3.so", ".abi3t.so")): | ||
| # abi3 and abi3t are not current co-installable (#122931) | ||
| return True | ||
|
|
||
| return False | ||
|
|
||
|
|
||
| def main() -> None: | ||
|
stefanor marked this conversation as resolved.
|
||
| p = ArgumentParser("Compare multiple hash-r files") | ||
| p.add_argument( | ||
| "base_directory", | ||
| type=Path, | ||
| help="Directory containing hashes of Python installs.", | ||
| ) | ||
| args = p.parse_args() | ||
| if not compare_trees(args.base_directory): | ||
| raise SystemExit(1) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is unrelated but is probably fine provided it's backported into the older branches.