diff --git a/.github/workflows/ci_run.yml b/.github/workflows/ci_run.yml index 507e254..0c91dd8 100644 --- a/.github/workflows/ci_run.yml +++ b/.github/workflows/ci_run.yml @@ -17,7 +17,7 @@ on: required: true type: string profile: - description: 'Dependency profile declared in ci/dependency-profiles.json' + description: 'Dependency profile declared in dependencies.toml' required: true type: string enable_reporting: @@ -128,11 +128,11 @@ jobs: with: node-version: 'lts/*' - - name: "CHECK: {Resolve dependency profile}" + - name: "CHECK: {Resolve dependencies}" id: dependencies run: | - METADATA="$RUNNER_TEMP/ci-dependencies.json" - python3 scripts/resolve-ci-dependencies.py resolve \ + METADATA="$RUNNER_TEMP/dependencies.json" + python3 scripts/resolve-dependencies.py resolve \ --profile '${{ inputs.profile }}' \ --output "$METADATA" \ --github-output "$GITHUB_OUTPUT" \ @@ -213,7 +213,7 @@ jobs: - name: "CHECK: {Verify dependency checkouts}" id: verified-dependencies run: | - python3 scripts/resolve-ci-dependencies.py verify \ + python3 scripts/resolve-dependencies.py verify \ --metadata "$CI_DEPENDENCY_METADATA" \ --github-output "$GITHUB_OUTPUT" diff --git a/README_ADVANCED.md b/README_ADVANCED.md index 161c3b2..34538b0 100644 --- a/README_ADVANCED.md +++ b/README_ADVANCED.md @@ -284,7 +284,10 @@ branch = "main" ### How Defaults Work -Defaults are defined in code (see [`src/config.rs`](src/config.rs) `Config::default()`) and written to `config.toml` during `init`. This means: +Operational defaults are defined in code (see [`src/config.rs`](src/config.rs) +`Config::default()`), while dependency locations come from +[`dependencies.toml`](dependencies.toml). They are written to +`config.toml` during `init`. This means: - **First-time setup:** Running `foc-devnet init` creates `config.toml` with current defaults from code - **Updating defaults:** When a new version of `foc-devnet` includes updated defaults (e.g., newer Lotus version), run `foc-devnet clean --all` then `foc-devnet init` to regenerate `config.toml` with the new defaults @@ -820,7 +823,9 @@ port_range_count = 100 - **[multicall3](https://github.com/mds1/multicall3)** - Multicall3 contract ### Dependent Version Strategy -Default versions for these repositories are defined in code (see [`src/config.rs`](src/config.rs) `Config::default()`). +Default versions for these repositories are defined in +[`dependencies.toml`](dependencies.toml) and consumed by +`Config::default()`. **Version specification methods:** - **Latest tag** (`latesttag`, `latesttag:`, `latesttag::`): Resolved once at `init` time via `git ls-remote` and pinned as a concrete `GitTag` in `config.toml`. Use a glob selector to scope which tags are considered, e.g. `latesttag:v*` or `latesttag:pdp/v*`. Bare `latesttag` matches all tags. @@ -1330,15 +1335,17 @@ Reports are written to `~/.foc-devnet/state/latest/scenario_report.md`. Scenarios run automatically in CI after the devnet starts. On nightly runs (or manual dispatch with `reporting` enabled), failures automatically create a GitHub issue with a full report. -CI resolves compatibility-sensitive dependencies from `ci/dependency-profiles.json`. +CI resolves compatibility-sensitive dependencies from `dependencies.toml`. Pull requests use the pinned `default` profile, while nightly `stability` runs use the latest final releases and nightly `frontier` runs pin current development branch heads to immutable commits. Nightly CI also runs manifest-declared mixed profiles such as `stability-frontier-curio`, where all dependencies come from `stability` except the named component, which comes from `frontier`. In `stability`, PDP comes from the stable filecoin-services checkout's bundled -submodule; in `frontier`, PDP is pinned as an independent repo. The resolved +submodule; in `frontier`, PDP is pinned as an independent repo. Mixed profiles +that override filecoin-services pin PDP to the configured bundled PDP commit so +the run does not unintentionally test two moving contract sources. The resolved metadata path is exposed to scenarios as `CI_DEPENDENCY_METADATA`; Synapse SDK -and filecoin-pin also receive their exact source, version/ref, and commit -through `SYNAPSE_SDK_*` and `FILECOIN_PIN_*` environment variables. PDP receives +and filecoin-pin receive their exact source, version/ref, and commit through +`SYNAPSE_SDK_*` and `FILECOIN_PIN_*` environment variables. PDP receives `PDP_SOURCE`, `PDP_REF`, and `PDP_COMMIT` when dependency profiles are resolved. diff --git a/build.rs b/build.rs index d07d515..b273e02 100644 --- a/build.rs +++ b/build.rs @@ -60,7 +60,8 @@ fn main() { println!("cargo:rerun-if-changed=.git/HEAD"); println!("cargo:rerun-if-changed=.git/refs/heads/"); - // Re-run if MockUSDFC contract files change + // Re-run if embedded defaults or MockUSDFC contract files change + println!("cargo:rerun-if-changed=dependencies.toml"); println!("cargo:rerun-if-changed=contracts/MockUSDFC/src/MockUSDFC.sol"); println!("cargo:rerun-if-changed=contracts/MockUSDFC/script/Deploy.s.sol"); println!("cargo:rerun-if-changed=contracts/MockUSDFC/script/Verify.s.sol"); diff --git a/ci/README.md b/ci/README.md index 9cf3f8a..ecd4ebf 100644 --- a/ci/README.md +++ b/ci/README.md @@ -1,11 +1,11 @@ # CI Dependency Profiles -`dependency-profiles.json` is the central manifest for CI dependency selection. -Its resolver is located in `scripts/resolve-ci-dependencies.py`. +`dependencies.toml` is the central manifest for CI dependency selection. +Its resolver is located in `scripts/resolve-dependencies.py`. ## Profiles -The manifest declares valid profiles in its top-level `profiles` object: +The manifest declares valid profiles in its top-level `profiles` tables: - `default`: used by PR CI, a known-working set of client versions. - `stability`: used by nightly CI to test stable releases. @@ -19,72 +19,88 @@ The manifest declares valid profiles in its top-level `profiles` object: - `stability-frontier-pdp`: used by nightly CI to test stable releases except PDP, which is resolved from `frontier`. -Each component must define a selection for every component profile referenced by -the top-level profile definitions. Today those component selections are -`default`, `stability`, and `frontier`. - -Top-level profile definitions have a `base` component profile and can override -specific components: - -```json -{ - "stability-frontier-curio": { - "base": "stability", - "components": { - "curio": "frontier" - } - } -} +Each dependency must define a selection for every profile selection referenced +by the top-level profile definitions. Today those selections are `default`, +`stability`, and `frontier`. + +Profile definitions have a `base` selection and can override specific +dependencies: + +```toml +[profiles.stability-frontier-curio] +base = "stability" + +[profiles.stability-frontier-curio.components] +curio = "frontier" ``` In that example, Curio resolves from its `frontier` selection while every other -component resolves from `stability`. A profile is valid only if it is explicitly +dependency resolves from `stability`. A profile is valid only if it is explicitly declared in `profiles`; for example, `stability-frontier-filecoin-pin` does not exist unless added there. -## Component Fields +The root manifest is also embedded by foc-devnet itself. Rust `Config::default()` +reads the runtime dependency locations it needs from the manifest's `default` +selections, so CI and local defaults share one source of truth. Scenario helpers +read their own dependency entries from the same manifest. -Top-level component fields: +## Dependency Fields + +Top-level dependency fields: - `repository`: Git repository URL. -- `npm_package`: npm package name, for components that are resolved through npm +- `npm_package`: npm package name, for dependencies that are resolved through npm metadata. -- `default`, `stability`, `frontier`: component profile selections. +- `default`, `stability`, `frontier`: dependency profile selections. + +Dependencies use named TOML tables so each dependency stays easy to review and +edit: + +```toml +[dependencies.lotus] +repository = "https://github.com/filecoin-project/lotus.git" + +default = { strategy = "git_tag", tag = "v1.36.1" } +stability = { strategy = "git_tag", tag = "v*" } +frontier = { strategy = "git_branch", branch = "master" } +``` Profile selections always have a `strategy`. Some strategies require additional fields. -### `config_default` +### `bundled` -Use the compiled `Config::default()` value and pass no runtime override to -`foc-devnet init`. +Use a git submodule bundled with a specific dependency selection. -```json -{ - "strategy": "config_default" -} +```toml +strategy = "bundled" +bundle = "filecoin-services@stability" +path = "service_contracts/lib/pdp" ``` +The `bundle` field uses `@`. When the active bundled +dependency selection matches that reference, the resolver omits an init override +and lets foc-devnet use the bundled submodule. When a mixed profile changes the +bundled dependency away from that selection, the resolver emits an explicit +`gitcommit:...` override for the bundled commit so the run does not test two +moving contract sources by accident. + ### `git_commit` Use an exact Git commit SHA. -```json -{ - "strategy": "git_commit", - "commit": "fadc836e65804311aca3bd2276861acabe42313f" -} +```toml +strategy = "git_commit" +commit = "fadc836e65804311aca3bd2276861acabe42313f" ``` ### `git_branch` Resolve a branch head to an immutable commit SHA before the run starts. -```json -{ - "strategy": "git_branch", - "branch": "master" -} +```toml +strategy = "git_branch" +branch = "master" ``` The resolved metadata records both the branch name and the exact commit. @@ -93,51 +109,41 @@ The resolved metadata records both the branch name and the exact commit. Resolve a Git tag to an immutable commit SHA. `tag` can be an exact tag: -```json -{ - "strategy": "git_tag", - "tag": "v1.2.3" -} +```toml +strategy = "git_tag" +tag = "v1.2.3" ``` `tag` can also be a pattern. Pattern selections choose the latest matching tag: -```json -{ - "strategy": "git_tag", - "tag": "v*" -} +```toml +strategy = "git_tag" +tag = "v*" ``` By default, pattern selections exclude prerelease tags such as `-rc`, `-alpha`, `-beta`, and development tags. Set `include_prereleases` to include them: -```json -{ - "strategy": "git_tag", - "tag": "v*", - "include_prereleases": true -} +```toml +strategy = "git_tag" +tag = "v*" +include_prereleases = true ``` ### `npm_version` Resolve an npm version, range, or dist-tag to a concrete package version. -```json -{ - "strategy": "npm_version", - "version": "1.0.1" -} +```toml +strategy = "npm_version" +version = "1.0.1" ``` The `version` field can also be an npm dist-tag: -```json -{ - "strategy": "npm_version", - "version": "latest" -} +```toml +strategy = "npm_version" +version = "latest" ``` The resolver records the concrete package version selected at resolution time @@ -149,17 +155,13 @@ Some profile selections can include an optional `overrides` object. Each entry maps a package name to a `version` and a `reason` explaining why the override exists: -```json -{ - "strategy": "git_tag", - "tag": "synapse-sdk-v1.0.1", - "overrides": { - "nanoid": { - "version": "3.3.13", - "reason": "nanoid 5.x is ESM-only and breaks the CJS build" - } - } -} +```toml +strategy = "git_tag" +tag = "synapse-sdk-v1.0.1" + +[dependencies.synapse-sdk.default.overrides.nanoid] +version = "3.3.13" +reason = "nanoid 5.x is ESM-only and breaks the CJS build" ``` Overrides are explicit profile policy. Both `version` and `reason` are required @@ -182,8 +184,8 @@ Current consumers: ## Current Boundary -`resolve-ci-dependencies.py` resolves metadata. It does **not** install -components. +`resolve-dependencies.py` resolves metadata. It does **not** install +dependencies. Installation currently lives in three places (which consume the resolved metadata): diff --git a/ci/dependency-profiles.json b/ci/dependency-profiles.json deleted file mode 100644 index 175dba1..0000000 --- a/ci/dependency-profiles.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "schema_version": 2, - "profiles": { - "default": { - "base": "default" - }, - "stability": { - "base": "stability" - }, - "frontier": { - "base": "frontier" - }, - "stability-frontier-lotus": { - "base": "stability", - "components": { - "lotus": "frontier" - } - }, - "stability-frontier-curio": { - "base": "stability", - "components": { - "curio": "frontier" - } - }, - "stability-frontier-filecoin-services": { - "base": "stability", - "components": { - "filecoin-services": "frontier" - } - }, - "stability-frontier-pdp": { - "base": "stability", - "components": { - "pdp": "frontier" - } - } - }, - "components": { - "lotus": { - "repository": "https://github.com/filecoin-project/lotus.git", - "default": { - "strategy": "config_default" - }, - "stability": { - "strategy": "git_tag", - "tag": "v*" - }, - "frontier": { - "strategy": "git_branch", - "branch": "master" - } - }, - "curio": { - "repository": "https://github.com/filecoin-project/curio.git", - "default": { - "strategy": "config_default" - }, - "stability": { - "strategy": "git_tag", - "tag": "v*" - }, - "frontier": { - "strategy": "git_branch", - "branch": "main" - } - }, - "filecoin-services": { - "repository": "https://github.com/FilOzone/filecoin-services.git", - "default": { - "strategy": "config_default" - }, - "stability": { - "strategy": "git_tag", - "tag": "v*" - }, - "frontier": { - "strategy": "git_branch", - "branch": "main" - } - }, - "pdp": { - "repository": "https://github.com/FilOzone/pdp.git", - "default": { - "strategy": "config_default" - }, - "stability": { - "strategy": "config_default" - }, - "frontier": { - "strategy": "git_branch", - "branch": "main" - } - }, - "synapse-sdk": { - "repository": "https://github.com/FilOzone/synapse-sdk.git", - "npm_package": "@filoz/synapse-sdk", - "default": { - "strategy": "git_tag", - "tag": "synapse-sdk-v1.1.0" - }, - "stability": { - "strategy": "git_tag", - "tag": "synapse-sdk-v*" - }, - "frontier": { - "strategy": "git_branch", - "branch": "master" - } - }, - "filecoin-pin": { - "repository": "https://github.com/filecoin-project/filecoin-pin.git", - "npm_package": "filecoin-pin", - "default": { - "strategy": "npm_version", - "version": "1.2.0" - }, - "stability": { - "strategy": "npm_version", - "version": "latest" - }, - "frontier": { - "strategy": "git_branch", - "branch": "master" - } - } - } -} diff --git a/dependencies.toml b/dependencies.toml new file mode 100644 index 0000000..c7fcdd7 --- /dev/null +++ b/dependencies.toml @@ -0,0 +1,101 @@ +schema_version = 2 + +[profiles.default] +base = "default" + +[profiles.stability] +base = "stability" + +[profiles.frontier] +base = "frontier" + +[profiles.stability-frontier-lotus] +base = "stability" + +[profiles.stability-frontier-lotus.components] +lotus = "frontier" + +[profiles.stability-frontier-curio] +base = "stability" + +[profiles.stability-frontier-curio.components] +curio = "frontier" + +[profiles.stability-frontier-filecoin-services] +base = "stability" + +[profiles.stability-frontier-filecoin-services.components] +filecoin-services = "frontier" + +[profiles.stability-frontier-pdp] +base = "stability" + +[profiles.stability-frontier-pdp.components] +pdp = "frontier" + +[dependencies.lotus] +repository = "https://github.com/filecoin-project/lotus.git" + +# renovate: datasource=github-tags depName=filecoin-project/lotus versioning=semver +default = { strategy = "git_tag", tag = "v1.36.1" } +stability = { strategy = "git_tag", tag = "v*" } +frontier = { strategy = "git_branch", branch = "master" } + +[dependencies.curio] +repository = "https://github.com/filecoin-project/curio.git" + +# renovate: datasource=github-tags depName=filecoin-project/curio versioning=semver +default = { strategy = "git_tag", tag = "v1.28.2" } +stability = { strategy = "git_tag", tag = "v*" } +frontier = { strategy = "git_branch", branch = "main" } + +[dependencies.filecoin-services] +repository = "https://github.com/FilOzone/filecoin-services.git" + +# renovate: datasource=github-tags depName=FilOzone/filecoin-services versioning=semver +default = { strategy = "git_tag", tag = "v1.3.0" } +stability = { strategy = "git_tag", tag = "v*" } +frontier = { strategy = "git_branch", branch = "main" } + +[dependencies.pdp] +repository = "https://github.com/FilOzone/pdp.git" + +[dependencies.pdp.default] +strategy = "bundled" +bundle = "filecoin-services@default" +path = "service_contracts/lib/pdp" + +[dependencies.pdp.stability] +strategy = "bundled" +bundle = "filecoin-services@stability" +path = "service_contracts/lib/pdp" + +[dependencies.pdp.frontier] +strategy = "git_branch" +branch = "main" + +[dependencies.multicall3] +repository = "https://github.com/mds1/multicall3.git" + +# renovate: datasource=github-tags depName=mds1/multicall3 versioning=semver +default = { strategy = "git_tag", tag = "v3.1.0" } +stability = { strategy = "git_tag", tag = "v3.1.0" } +frontier = { strategy = "git_tag", tag = "v3.1.0" } + +[dependencies.synapse-sdk] +repository = "https://github.com/FilOzone/synapse-sdk.git" +npm_package = "@filoz/synapse-sdk" + +# renovate: datasource=github-tags depName=FilOzone/synapse-sdk extractVersion=^synapse-sdk-v(?.+)$ versioning=semver +default = { strategy = "git_tag", tag = "synapse-sdk-v1.1.0" } +stability = { strategy = "git_tag", tag = "synapse-sdk-v*" } +frontier = { strategy = "git_branch", branch = "master" } + +[dependencies.filecoin-pin] +repository = "https://github.com/filecoin-project/filecoin-pin.git" +npm_package = "filecoin-pin" + +# renovate: datasource=npm depName=filecoin-pin versioning=npm +default = { strategy = "npm_version", version = "1.2.0" } +stability = { strategy = "npm_version", version = "latest" } +frontier = { strategy = "git_branch", branch = "master" } diff --git a/renovate.json b/renovate.json index 34ed648..c7d7b63 100644 --- a/renovate.json +++ b/renovate.json @@ -7,10 +7,10 @@ { "customType": "regex", "managerFilePatterns": [ - "/^src/config\\.rs$/" + "/^dependencies\\.toml$/" ], "matchStrings": [ - "url:\\s*\"https://github\\.com/(?[^\"]+?)(?:\\.git)?\"\\.to_string\\(\\),\\s*tag:\\s*\"(?v[0-9][^\"]*)\"" + "# renovate: datasource=github-tags depName=(?\\S+) versioning=semver\\s+default\\s*=\\s*\\{[^\\n]*tag\\s*=\\s*\"(?v[0-9][^\"]*)\"" ], "datasourceTemplate": "github-tags", "versioningTemplate": "semver" @@ -18,10 +18,10 @@ { "customType": "regex", "managerFilePatterns": [ - "/^ci/dependency-profiles\\.json$/" + "/^dependencies\\.toml$/" ], "matchStrings": [ - "\"tag\":\\s*\"synapse-sdk-v(?[0-9][^\"]*)\"" + "# renovate: datasource=github-tags depName=FilOzone/synapse-sdk extractVersion=\\^synapse-sdk-v\\(\\?\\.\\+\\)\\$ versioning=semver\\s+default\\s*=\\s*\\{[^\\n]*tag\\s*=\\s*\"(?synapse-sdk-v[^\"]*)\"" ], "depNameTemplate": "FilOzone/synapse-sdk", "datasourceTemplate": "github-tags", @@ -31,10 +31,10 @@ { "customType": "regex", "managerFilePatterns": [ - "/^ci/dependency-profiles\\.json$/" + "/^dependencies\\.toml$/" ], "matchStrings": [ - "\"strategy\":\\s*\"npm_version\",\\s*\"version\":\\s*\"(?[0-9][^\"]*)\"" + "# renovate: datasource=npm depName=filecoin-pin versioning=npm\\s+default\\s*=\\s*\\{[^\\n]*version\\s*=\\s*\"(?[0-9][^\"]*)\"" ], "depNameTemplate": "filecoin-pin", "datasourceTemplate": "npm", diff --git a/scenarios/dependencies.py b/scenarios/dependencies.py index e9e5d2b..a4242e7 100644 --- a/scenarios/dependencies.py +++ b/scenarios/dependencies.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 -"""Access resolved CI dependency metadata from scenario tests.""" +"""Access resolved dependency metadata from scenario tests.""" from __future__ import annotations import json import os +import tomllib from pathlib import Path @@ -27,11 +28,11 @@ def component(name: str) -> dict: if isinstance(value, dict): return value - manifest_path = Path(__file__).parents[1] / "ci" / "dependency-profiles.json" - manifest = json.loads(manifest_path.read_text()) - definition = manifest["components"].get(name) + manifest_path = Path(__file__).parents[1] / "dependencies.toml" + manifest = tomllib.loads(manifest_path.read_text()) + definition = manifest["dependencies"].get(name) if not definition: - raise RuntimeError(f"Dependency manifest has no {name!r} component") + raise RuntimeError(f"Dependency manifest has no {name!r} dependency") selection = definition["default"] fallback = { "name": name, diff --git a/scripts/resolve-ci-dependencies.py b/scripts/resolve-dependencies.py similarity index 69% rename from scripts/resolve-ci-dependencies.py rename to scripts/resolve-dependencies.py index 474ad2d..218da96 100644 --- a/scripts/resolve-ci-dependencies.py +++ b/scripts/resolve-dependencies.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Resolve CI dependency profiles to immutable versions and verify checkouts.""" +"""Resolve dependency profiles to immutable versions and verify checkouts.""" from __future__ import annotations @@ -7,19 +7,24 @@ import fnmatch import hashlib import json -import os import re import shlex import subprocess +import tempfile +import tomllib from pathlib import Path MANIFEST_SCHEMA_VERSION = 2 +PDP_DEFAULT_BUNDLE_PATH = "service_contracts/lib/pdp" INIT_COMPONENT_FLAGS = { "lotus": "--lotus", "curio": "--curio", "filecoin-services": "--filecoin-services", "pdp": "--pdp", } +RUNTIME_COMPONENTS = set(INIT_COMPONENT_FLAGS) | {"multicall3"} +TEST_COMPONENTS = {"synapse-sdk", "filecoin-pin"} +REQUIRED_COMPONENTS = RUNTIME_COMPONENTS | TEST_COMPONENTS VERSION_RE = re.compile(r"^\d+(?:\.\d+)*(?:[-+][0-9A-Za-z.-]+)?$") STABLE_VERSION_RE = re.compile(r"^\d+(?:\.\d+)*$") COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") @@ -39,8 +44,8 @@ def run_command(command: list[str]) -> str: def load_manifest(path: Path) -> dict: try: - manifest = json.loads(path.read_text()) - except (OSError, json.JSONDecodeError) as error: + manifest = tomllib.loads(path.read_text()) + except (OSError, tomllib.TOMLDecodeError) as error: raise ResolutionError( f"Cannot load dependency manifest {path}: {error}" ) from error @@ -49,22 +54,65 @@ def load_manifest(path: Path) -> dict: raise ResolutionError( f"Dependency manifest schema_version must be {MANIFEST_SCHEMA_VERSION}" ) - components = manifest.get("components") - if not isinstance(components, dict): - raise ResolutionError("Dependency manifest must contain a components object") + dependencies = manifest.get("dependencies") + if not isinstance(dependencies, dict): + raise ResolutionError("Dependency manifest must contain a dependencies object") profiles = manifest.get("profiles") if not isinstance(profiles, dict): raise ResolutionError("Dependency manifest must contain a profiles object") - required = set(INIT_COMPONENT_FLAGS) | {"synapse-sdk", "filecoin-pin"} - missing = sorted(required - set(components)) + missing = sorted(REQUIRED_COMPONENTS - set(dependencies)) if missing: - raise ResolutionError(f"Dependency manifest is missing: {', '.join(missing)}") - validate_profiles(profiles, components) + raise ResolutionError( + "Dependency manifest is missing dependencies: " + ", ".join(missing) + ) + validate_profiles(profiles, dependencies) + validate_bundled_selections(dependencies) return manifest -def validate_profiles(profiles: dict, components: dict) -> None: +def parse_bundle_ref(bundle: str) -> tuple[str, str]: + if not isinstance(bundle, str) or not bundle: + raise ResolutionError("bundled strategy requires a non-empty bundle") + dependency, separator, selection = bundle.partition("@") + if not separator or not dependency or not selection or "@" in selection: + raise ResolutionError( + "bundled strategy bundle must use '@'" + ) + return dependency, selection + + +def validate_bundled_selections(components: dict) -> None: + for component_name, component in components.items(): + if not isinstance(component, dict): + continue + for selection_name, selection in component.items(): + if selection_name in {"repository", "npm_package"}: + continue + if not isinstance(selection, dict): + continue + if selection.get("strategy") != "bundled": + continue + dependency, bundled_selection = parse_bundle_ref(selection.get("bundle")) + bundled_component = components.get(dependency) + if not isinstance(bundled_component, dict): + raise ResolutionError( + f"{component_name} bundled selection references unknown " + f"dependency {dependency!r}" + ) + if bundled_selection not in bundled_component: + raise ResolutionError( + f"{component_name} bundled selection references unknown " + f"{dependency} selection {bundled_selection!r}" + ) + path = selection.get("path", PDP_DEFAULT_BUNDLE_PATH) + if not isinstance(path, str) or not path: + raise ResolutionError("bundled strategy path must be a string") + if path.startswith("/") or ".." in Path(path).parts: + raise ResolutionError("bundled strategy path must be relative") + + +def validate_profiles(profiles: dict, dependencies: dict) -> None: for profile_name, profile in profiles.items(): if not isinstance(profile_name, str) or not profile_name: raise ResolutionError("Profile names must be non-empty strings") @@ -81,14 +129,14 @@ def validate_profiles(profiles: dict, components: dict) -> None: f"Profile {profile_name!r} components must be an object" ) - unknown_components = sorted(set(component_overrides) - set(components)) + unknown_components = sorted(set(component_overrides) - set(dependencies)) if unknown_components: raise ResolutionError( - f"Profile {profile_name!r} references unknown components: " + f"Profile {profile_name!r} references unknown dependencies: " f"{', '.join(unknown_components)}" ) - for component_name, component in components.items(): + for component_name, component in dependencies.items(): selection_profile = component_overrides.get(component_name, base) if not isinstance(selection_profile, str) or not selection_profile: raise ResolutionError( @@ -98,7 +146,7 @@ def validate_profiles(profiles: dict, components: dict) -> None: if selection_profile not in component: raise ResolutionError( f"Profile {profile_name!r} selects {selection_profile!r} for " - f"{component_name}, but that component has no such selection" + f"{component_name}, but that dependency has no such selection" ) @@ -112,7 +160,7 @@ def component_profile_map(manifest: dict, profile_name: str) -> dict[str, str]: component_overrides = profile.get("components", {}) return { component_name: component_overrides.get(component_name, base) - for component_name in manifest["components"] + for component_name in manifest["dependencies"] } @@ -272,7 +320,11 @@ def validate_overrides(name: str, strategy: str, overrides) -> dict: def resolve_component( - name: str, component: dict, profile: str, runner=run_command + name: str, + component: dict, + profile: str, + manifest: dict | None = None, + runner=run_command, ) -> dict: try: selection = component[profile] @@ -288,9 +340,7 @@ def resolve_component( "strategy": strategy, } - if strategy == "config_default": - resolved["source"] = "config_default" - elif strategy == "git_commit": + if strategy == "git_commit": commit = selection["commit"] if not COMMIT_RE.fullmatch(commit): raise ResolutionError(f"{name} has invalid commit SHA {commit!r}") @@ -319,6 +369,45 @@ def resolve_component( version=data["version"], commit=data.get("gitHead", ""), ) + elif strategy == "bundled": + if manifest is None: + raise ResolutionError("bundled strategy requires the full manifest") + dependency, bundled_selection = parse_bundle_ref(selection["bundle"]) + bundle_path = selection.get("path", PDP_DEFAULT_BUNDLE_PATH) + dependencies = manifest["dependencies"] + if dependency not in dependencies: + raise ResolutionError( + f"{name} bundled selection references unknown dependency " + f"{dependency!r}" + ) + bundle_source = resolve_component( + dependency, + dependencies[dependency], + bundled_selection, + manifest, + runner, + ) + bundled_commit = read_gitlink( + bundle_source["repository"], + bundle_source["commit"], + bundle_path, + runner, + ) + resolved.update( + source="bundled", + ref_type="commit", + ref=bundled_commit, + commit=bundled_commit, + bundled_from={ + "dependency": dependency, + "selection_profile": bundled_selection, + "repository": bundle_source["repository"], + "ref_type": bundle_source["ref_type"], + "ref": bundle_source["ref"], + "commit": bundle_source["commit"], + "path": bundle_path, + }, + ) else: raise ResolutionError(f"Unsupported strategy {strategy!r} for {name}") overrides = selection.get("overrides") @@ -327,11 +416,34 @@ def resolve_component( return resolved +def read_gitlink(repository: str, commit: str, path: str, runner=run_command) -> str: + with tempfile.TemporaryDirectory(prefix="foc-devnet-ci-deps-") as directory: + repo_dir = Path(directory) / "repo" + runner(["git", "-C", directory, "init", "repo"]) + runner(["git", "-C", str(repo_dir), "remote", "add", "origin", repository]) + runner(["git", "-C", str(repo_dir), "fetch", "--depth=1", "origin", commit]) + output = runner(["git", "-C", str(repo_dir), "ls-tree", "FETCH_HEAD", path]) + + fields = output.split() + if len(fields) < 4 or fields[0] != "160000" or fields[1] != "commit": + raise ResolutionError( + f"{path} in {repository}@{commit} is not a git submodule gitlink" + ) + gitlink = fields[2] + if not COMMIT_RE.fullmatch(gitlink): + raise ResolutionError( + f"{path} in {repository}@{commit} has invalid gitlink SHA {gitlink!r}" + ) + return gitlink + + def build_init_args(components: dict) -> list[str]: args = [] for name, flag in INIT_COMPONENT_FLAGS.items(): component = components[name] - if component["source"] == "config_default": + if component_uses_active_bundle(components, name): + continue + if component["selection_profile"] == "default": continue args.extend( [ @@ -342,6 +454,19 @@ def build_init_args(components: dict) -> list[str]: return args +def component_uses_active_bundle(components: dict, name: str) -> bool: + component = components[name] + if component["source"] != "bundled": + return False + bundled_from = component["bundled_from"] + dependency = bundled_from["dependency"] + if dependency not in components: + return False + return ( + components[dependency]["selection_profile"] == bundled_from["selection_profile"] + ) + + def cache_hash(components: dict) -> str: values = [f"{name}:{components[name]['commit']}" for name in ("lotus", "curio")] return hashlib.sha256("\n".join(values).encode()).hexdigest() @@ -375,10 +500,11 @@ def scenario_environment(metadata_path: Path, components: dict) -> dict: def resolve(args) -> None: manifest = load_manifest(args.manifest) component_profiles = component_profile_map(manifest, args.profile) - components = { - name: resolve_component(name, component, component_profiles[name]) - for name, component in manifest["components"].items() - } + components = {} + for name, component in manifest["dependencies"].items(): + components[name] = resolve_component( + name, component, component_profiles[name], manifest, run_command + ) metadata = { "schema_version": 1, "profile": args.profile, @@ -397,7 +523,7 @@ def verify(args) -> None: metadata = json.loads(args.metadata.read_text()) components = metadata["components"] for name in INIT_COMPONENT_FLAGS: - if name == "pdp" and components[name]["source"] == "config_default": + if component_uses_active_bundle(components, name): continue repository_path = args.code_dir / name actual = run_command(["git", "-C", str(repository_path), "rev-parse", "HEAD"]) @@ -423,7 +549,7 @@ def parser() -> argparse.ArgumentParser: resolve_parser = subparsers.add_parser("resolve") resolve_parser.add_argument("--profile", required=True) resolve_parser.add_argument( - "--manifest", type=Path, default=Path("ci/dependency-profiles.json") + "--manifest", type=Path, default=Path("dependencies.toml") ) resolve_parser.add_argument("--output", type=Path, required=True) resolve_parser.add_argument("--github-output") diff --git a/scripts/tests/test_resolve_ci_dependencies.py b/scripts/tests/test_resolve_dependencies.py similarity index 60% rename from scripts/tests/test_resolve_ci_dependencies.py rename to scripts/tests/test_resolve_dependencies.py index 26bd147..fc71234 100644 --- a/scripts/tests/test_resolve_ci_dependencies.py +++ b/scripts/tests/test_resolve_dependencies.py @@ -7,12 +7,49 @@ from pathlib import Path from unittest.mock import patch -SCRIPT = Path(__file__).parents[1] / "resolve-ci-dependencies.py" +SCRIPT = Path(__file__).parents[1] / "resolve-dependencies.py" SPEC = importlib.util.spec_from_file_location("dependency_resolver", SCRIPT) resolver = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(resolver) +def toml_scalar(value): + if isinstance(value, bool): + return str(value).lower() + if isinstance(value, str): + return json.dumps(value) + if isinstance(value, int): + return str(value) + raise TypeError(f"Unsupported TOML fixture value: {value!r}") + + +def toml_key(key): + return key + + +def toml_document(data): + lines = [] + + def emit_table(path, table): + scalars = { + key: value for key, value in table.items() if not isinstance(value, dict) + } + children = { + key: value for key, value in table.items() if isinstance(value, dict) + } + + if path: + lines.append("") + lines.append(f"[{'.'.join(toml_key(part) for part in path)}]") + for key, value in scalars.items(): + lines.append(f"{toml_key(key)} = {toml_scalar(value)}") + for key, value in children.items(): + emit_table([*path, key], value) + + emit_table([], data) + return "\n".join(lines).lstrip() + "\n" + + class FakeRunner: def __init__(self, responses): self.responses = ( @@ -38,9 +75,14 @@ def manifest(self, profiles=None, components=None): "curio": self.component("c", "d"), "filecoin-services": self.component("e", "f"), "pdp": self.component("5", "6"), + "multicall3": self.component("7", "8"), + } + components.update( + { "synapse-sdk": self.component("1", "2"), "filecoin-pin": self.component("3", "4"), } + ) if profiles is None: profiles = { "default": {"base": "default"}, @@ -66,13 +108,16 @@ def manifest(self, profiles=None, components=None): return { "schema_version": 2, "profiles": profiles, - "components": components, + "dependencies": components, } def component(self, stability_prefix, frontier_prefix): return { "repository": "https://example.test/project.git", - "default": {"strategy": "config_default"}, + "default": { + "strategy": "git_commit", + "commit": "0" * 40, + }, "stability": { "strategy": "git_commit", "commit": stability_prefix * 40, @@ -86,9 +131,9 @@ def component(self, stability_prefix, frontier_prefix): def resolve_manifest(self, manifest, profile): with tempfile.TemporaryDirectory() as directory: directory = Path(directory) - manifest_path = directory / "manifest.json" + manifest_path = directory / "manifest.toml" output_path = directory / "resolved.json" - manifest_path.write_text(json.dumps(manifest)) + manifest_path.write_text(toml_document(manifest)) args = type( "Args", (), @@ -104,6 +149,50 @@ def resolve_manifest(self, manifest, profile): resolver.resolve(args) return json.loads(output_path.read_text()) + def resolve_manifest_with_github_output(self, manifest, profile): + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + manifest_path = directory / "manifest.toml" + output_path = directory / "resolved.json" + github_output_path = directory / "github-output" + manifest_path.write_text(toml_document(manifest)) + args = type( + "Args", + (), + { + "profile": profile, + "manifest": manifest_path, + "output": output_path, + "github_output": str(github_output_path), + "github_env": None, + }, + ) + with redirect_stdout(StringIO()): + resolver.resolve(args) + return ( + json.loads(output_path.read_text()), + github_output_path.read_text(), + ) + + def pdp_bundled_component(self): + return { + "repository": "https://example.test/pdp.git", + "default": { + "strategy": "bundled", + "bundle": "filecoin-services@default", + "path": "service_contracts/lib/pdp", + }, + "stability": { + "strategy": "bundled", + "bundle": "filecoin-services@stability", + "path": "service_contracts/lib/pdp", + }, + "frontier": { + "strategy": "git_commit", + "commit": "6" * 40, + }, + } + def test_latest_non_prerelease_tag_excludes_prereleases_and_annotated_refs(self): output = "\n".join( [ @@ -159,7 +248,9 @@ def test_git_tag_pattern_can_include_prereleases(self): } ) - resolved = resolver.resolve_component("project", component, "stability", runner) + resolved = resolver.resolve_component( + "project", component, "stability", runner=runner + ) self.assertEqual(resolved["ref"], "v2.0.0-rc1") self.assertEqual(resolved["commit"], "bbb") @@ -168,9 +259,9 @@ def test_unknown_profile_fails(self): manifest = self.manifest() with tempfile.TemporaryDirectory() as directory: directory = Path(directory) - manifest_path = directory / "manifest.json" + manifest_path = directory / "manifest.toml" output_path = directory / "resolved.json" - manifest_path.write_text(json.dumps(manifest)) + manifest_path.write_text(toml_document(manifest)) args = type( "Args", (), @@ -187,16 +278,19 @@ def test_unknown_profile_fails(self): def test_manifest_missing_component_fails(self): with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "manifest.json" - path.write_text( - json.dumps( - { - "schema_version": 2, - "profiles": {"default": {"base": "default"}}, - "components": {}, - } - ) - ) + path = Path(directory) / "manifest.toml" + path.write_text(""" +schema_version = 2 + +[profiles.default.runtime] +base = "default" + +[profiles.default.test] +base = "default" + +[dependencies] + +""") with self.assertRaisesRegex(resolver.ResolutionError, "missing"): resolver.load_manifest(path) @@ -216,8 +310,74 @@ def test_mixed_profile_resolves_only_selected_component_from_frontier(self): self.assertEqual(components["synapse-sdk"]["selection_profile"], "stability") self.assertEqual(components["filecoin-pin"]["selection_profile"], "stability") + @patch.object(resolver, "run_command") + def test_filecoin_services_mixed_profile_pins_default_bundled_pdp( + self, run_command + ): + pdp_commit = "7" * 40 + manifest = self.manifest() + manifest["dependencies"]["pdp"] = self.pdp_bundled_component() + run_command.side_effect = FakeRunner( + [ + ( + lambda command: command[:3] == ["git", "-C", command[2]] + and command[3:] == ["init", "repo"], + "", + ), + ( + lambda command: command[:5] + == ["git", "-C", command[2], "remote", "add"], + "", + ), + ( + lambda command: command[:4] == ["git", "-C", command[2], "fetch"], + "", + ), + ( + lambda command: command[:4] == ["git", "-C", command[2], "ls-tree"], + f"160000 commit {pdp_commit}\tservice_contracts/lib/pdp", + ), + ] + ) + + metadata, github_output = self.resolve_manifest_with_github_output( + manifest, "stability-frontier-filecoin-services" + ) + components = metadata["components"] + + self.assertEqual( + components["filecoin-services"]["selection_profile"], "frontier" + ) + self.assertEqual(components["filecoin-services"]["commit"], "f" * 40) + self.assertEqual(components["pdp"]["selection_profile"], "stability") + self.assertEqual(components["pdp"]["source"], "bundled") + self.assertEqual(components["pdp"]["commit"], pdp_commit) + self.assertEqual( + components["pdp"]["bundled_from"], + { + "dependency": "filecoin-services", + "selection_profile": "stability", + "repository": "https://example.test/project.git", + "ref_type": "commit", + "ref": "e" * 40, + "commit": "e" * 40, + "path": "service_contracts/lib/pdp", + }, + ) + self.assertIn( + "--filecoin-services gitcommit:https://example.test/project.git:" + + "f" * 40, + github_output, + ) + self.assertIn( + "--pdp gitcommit:https://example.test/pdp.git:" + pdp_commit, + github_output, + ) + def test_pdp_mixed_profile_resolves_only_pdp_from_frontier(self): - metadata = self.resolve_manifest(self.manifest(), "stability-frontier-pdp") + manifest = self.manifest() + manifest["dependencies"]["pdp"] = self.pdp_bundled_component() + metadata = self.resolve_manifest(manifest, "stability-frontier-pdp") components = metadata["components"] self.assertEqual(metadata["profile"], "stability-frontier-pdp") @@ -231,6 +391,126 @@ def test_pdp_mixed_profile_resolves_only_pdp_from_frontier(self): self.assertEqual(components["synapse-sdk"]["selection_profile"], "stability") self.assertEqual(components["filecoin-pin"]["selection_profile"], "stability") + def test_frontier_profile_uses_independent_pdp_frontier(self): + manifest = self.manifest() + manifest["dependencies"]["pdp"] = self.pdp_bundled_component() + + metadata, github_output = self.resolve_manifest_with_github_output( + manifest, "frontier" + ) + + self.assertEqual(metadata["components"]["pdp"]["selection_profile"], "frontier") + self.assertEqual(metadata["components"]["pdp"]["source"], "git") + self.assertEqual(metadata["components"]["pdp"]["commit"], "6" * 40) + self.assertIn( + "--pdp gitcommit:https://example.test/pdp.git:" + "6" * 40, + github_output, + ) + + @patch.object(resolver, "run_command") + def test_bundled_pdp_omits_init_arg_for_stability_profile(self, run_command): + pdp_commit = "7" * 40 + manifest = self.manifest() + manifest["dependencies"]["pdp"] = self.pdp_bundled_component() + run_command.side_effect = FakeRunner( + [ + ( + lambda command: command[:3] == ["git", "-C", command[2]] + and command[3:] == ["init", "repo"], + "", + ), + ( + lambda command: command[:5] + == ["git", "-C", command[2], "remote", "add"], + "", + ), + ( + lambda command: command[:4] == ["git", "-C", command[2], "fetch"], + "", + ), + ( + lambda command: command[:4] == ["git", "-C", command[2], "ls-tree"], + f"160000 commit {pdp_commit}\tservice_contracts/lib/pdp", + ), + ] + ) + + metadata, github_output = self.resolve_manifest_with_github_output( + manifest, "stability" + ) + + self.assertEqual(metadata["components"]["pdp"]["source"], "bundled") + self.assertEqual(metadata["components"]["pdp"]["commit"], pdp_commit) + self.assertNotIn("--pdp", github_output) + + @patch.object(resolver, "run_command") + def test_bundled_pdp_omits_init_arg_for_default_profile(self, run_command): + pdp_commit = "7" * 40 + manifest = self.manifest() + manifest["dependencies"]["pdp"] = self.pdp_bundled_component() + run_command.side_effect = FakeRunner( + [ + ( + lambda command: command[:3] == ["git", "-C", command[2]] + and command[3:] == ["init", "repo"], + "", + ), + ( + lambda command: command[:5] + == ["git", "-C", command[2], "remote", "add"], + "", + ), + ( + lambda command: command[:4] == ["git", "-C", command[2], "fetch"], + "", + ), + ( + lambda command: command[:4] == ["git", "-C", command[2], "ls-tree"], + f"160000 commit {pdp_commit}\tservice_contracts/lib/pdp", + ), + ] + ) + + metadata, github_output = self.resolve_manifest_with_github_output( + manifest, "default" + ) + + self.assertEqual(metadata["components"]["pdp"]["source"], "bundled") + self.assertEqual(metadata["components"]["pdp"]["commit"], pdp_commit) + self.assertEqual(github_output, "init-args=\n") + + @patch.object(resolver, "run_command") + def test_filecoin_services_mixed_profile_rejects_missing_pdp_gitlink( + self, run_command + ): + manifest = self.manifest() + manifest["dependencies"]["pdp"] = self.pdp_bundled_component() + run_command.side_effect = FakeRunner( + [ + ( + lambda command: command[:3] == ["git", "-C", command[2]] + and command[3:] == ["init", "repo"], + "", + ), + ( + lambda command: command[:5] + == ["git", "-C", command[2], "remote", "add"], + "", + ), + ( + lambda command: command[:4] == ["git", "-C", command[2], "fetch"], + "", + ), + ( + lambda command: command[:4] == ["git", "-C", command[2], "ls-tree"], + "100644 blob abcdef\tservice_contracts/lib/pdp", + ), + ] + ) + + with self.assertRaisesRegex(resolver.ResolutionError, "not a git submodule"): + self.resolve_manifest(manifest, "stability-frontier-filecoin-services") + def test_absent_mixed_profile_is_rejected(self): profiles = { "default": {"base": "default"}, @@ -256,9 +536,11 @@ def test_manifest_profile_rejects_unknown_component_override(self): } ) with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "manifest.json" - path.write_text(json.dumps(manifest)) - with self.assertRaisesRegex(resolver.ResolutionError, "unknown components"): + path = Path(directory) / "manifest.toml" + path.write_text(toml_document(manifest)) + with self.assertRaisesRegex( + resolver.ResolutionError, "unknown dependencies" + ): resolver.load_manifest(path) def test_manifest_profile_rejects_missing_component_selection(self): @@ -272,8 +554,8 @@ def test_manifest_profile_rejects_missing_component_selection(self): } ) with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "manifest.json" - path.write_text(json.dumps(manifest)) + path = Path(directory) / "manifest.toml" + path.write_text(toml_document(manifest)) with self.assertRaisesRegex(resolver.ResolutionError, "no such selection"): resolver.load_manifest(path) @@ -303,7 +585,7 @@ def test_npm_version_resolves_dist_tag_to_npm_version(self): ) resolved = resolver.resolve_component( - "filecoin-pin", component, "stability", runner + "filecoin-pin", component, "stability", runner=runner ) self.assertEqual(resolved["source"], "npm") @@ -339,7 +621,7 @@ def test_profile_overrides_are_copied_to_resolved_component(self): ) resolved = resolver.resolve_component( - "filecoin-pin", component, "default", runner + "filecoin-pin", component, "default", runner=runner ) self.assertEqual( @@ -377,7 +659,9 @@ def test_profile_overrides_require_version_and_reason(self): ) with self.assertRaisesRegex(resolver.ResolutionError, "reason"): - resolver.resolve_component("filecoin-pin", component, "default", runner) + resolver.resolve_component( + "filecoin-pin", component, "default", runner=runner + ) def test_overrides_are_rejected_for_core_git_components(self): component = { @@ -400,7 +684,7 @@ def test_overrides_are_rejected_for_core_git_components(self): ) with self.assertRaisesRegex(resolver.ResolutionError, "not supported"): - resolver.resolve_component("lotus", component, "frontier", runner) + resolver.resolve_component("lotus", component, "frontier", runner=runner) def test_overrides_are_rejected_for_source_built_filecoin_pin(self): component = { @@ -423,7 +707,9 @@ def test_overrides_are_rejected_for_source_built_filecoin_pin(self): ) with self.assertRaisesRegex(resolver.ResolutionError, "not supported"): - resolver.resolve_component("filecoin-pin", component, "frontier", runner) + resolver.resolve_component( + "filecoin-pin", component, "frontier", runner=runner + ) def test_frontier_branch_resolves_to_commit(self): component = { @@ -440,7 +726,9 @@ def test_frontier_branch_resolves_to_commit(self): ): "deadbeef refs/heads/main", } ) - resolved = resolver.resolve_component("project", component, "frontier", runner) + resolved = resolver.resolve_component( + "project", component, "frontier", runner=runner + ) self.assertEqual(resolved["commit"], "deadbeef") def test_git_commit_strategy_uses_exact_sha_without_resolution(self): @@ -450,7 +738,7 @@ def test_git_commit_strategy_uses_exact_sha_without_resolution(self): "default": {"strategy": "git_commit", "commit": commit}, } resolved = resolver.resolve_component( - "synapse-sdk", component, "default", FakeRunner({}) + "synapse-sdk", component, "default", runner=FakeRunner({}) ) self.assertEqual(resolved["ref_type"], "commit") self.assertEqual(resolved["ref"], commit) @@ -463,24 +751,27 @@ def test_git_commit_strategy_rejects_non_sha(self): } with self.assertRaisesRegex(resolver.ResolutionError, "invalid commit SHA"): resolver.resolve_component( - "synapse-sdk", component, "default", FakeRunner({}) + "synapse-sdk", component, "default", runner=FakeRunner({}) ) - def test_init_args_skip_config_defaults_and_pin_other_sources(self): + def test_init_args_skip_default_selections_and_pin_other_sources(self): components = { - "lotus": {"source": "config_default"}, + "lotus": {"source": "git", "selection_profile": "default"}, "curio": { "source": "git", + "selection_profile": "frontier", "repository": "https://example.test/curio.git", "commit": "abc", }, "filecoin-services": { "source": "git", + "selection_profile": "stability", "repository": "https://example.test/services.git", "commit": "def", }, "pdp": { "source": "git", + "selection_profile": "frontier", "repository": "https://example.test/pdp.git", "commit": "123", }, @@ -497,12 +788,10 @@ def test_init_args_skip_config_defaults_and_pin_other_sources(self): ], ) - def test_init_args_skip_default_pdp(self): + def test_init_args_skip_default_selections(self): components = { - "lotus": {"source": "config_default"}, - "curio": {"source": "config_default"}, - "filecoin-services": {"source": "config_default"}, - "pdp": {"source": "config_default"}, + name: {"source": "git", "selection_profile": "default"} + for name in ("lotus", "curio", "filecoin-services", "pdp") } self.assertEqual(resolver.build_init_args(components), []) @@ -530,8 +819,23 @@ def test_verify_records_checkouts_and_writes_cache_key(self, run_command): "schema_version": 1, "profile": "default", "components": { - **{name: {"source": "config_default"} for name in commits}, - "pdp": {"source": "config_default"}, + **{ + name: { + "source": "git", + "selection_profile": "default", + "commit": commit, + } + for name, commit in commits.items() + }, + "pdp": { + "source": "bundled", + "selection_profile": "default", + "commit": "ddd", + "bundled_from": { + "dependency": "filecoin-services", + "selection_profile": "default", + }, + }, }, } with tempfile.TemporaryDirectory() as directory: @@ -570,7 +874,11 @@ def test_verify_checks_independent_pdp_checkout(self, run_command): "schema_version": 1, "profile": "frontier", "components": { - name: {"source": "git", "commit": commit} + name: { + "source": "git", + "selection_profile": "frontier", + "commit": commit, + } for name, commit in commits.items() }, } diff --git a/src/config.rs b/src/config.rs index 17706f5..b0dff31 100644 --- a/src/config.rs +++ b/src/config.rs @@ -5,6 +5,9 @@ //! port allocations, and executable locations for various components. use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +const DEPENDENCIES_TOML: &str = include_str!("../dependencies.toml"); /// Represents the location of an executable or source code for a component. /// @@ -281,33 +284,123 @@ impl Default for Config { /// The defaults should always use `GitCommit` or `GitTag` locations to ensure /// reproducibility. fn default() -> Self { - Self { + Self::default_from_dependency_manifest(DEPENDENCIES_TOML) + .expect("embedded dependencies.toml must define valid defaults") + } +} + +#[derive(Debug, Deserialize)] +struct DependencyManifest { + schema_version: u8, + dependencies: HashMap, +} + +#[derive(Debug, Deserialize)] +struct DependencyComponent { + repository: String, + default: DependencySelection, +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "strategy", rename_all = "snake_case")] +enum DependencySelection { + GitCommit { commit: String }, + GitTag { tag: String }, + GitBranch { branch: String }, + Bundled { bundle: String, path: String }, + NpmVersion { version: String }, +} + +impl Config { + fn default_from_dependency_manifest(manifest_toml: &str) -> Result { + let manifest: DependencyManifest = toml::from_str(manifest_toml) + .map_err(|error| format!("Failed to parse dependency manifest: {}", error))?; + if manifest.schema_version != 2 { + return Err(format!( + "Dependency manifest schema_version must be 2, found {}", + manifest.schema_version + )); + } + + Ok(Self { port_range_start: 5700, port_range_count: 100, - lotus: Location::GitTag { - url: "https://github.com/filecoin-project/lotus.git".to_string(), - tag: "v1.36.1".to_string(), - }, - curio: Location::GitTag { - url: "https://github.com/filecoin-project/curio.git".to_string(), - tag: "v1.28.2".to_string(), - }, - filecoin_services: Location::GitTag { - url: "https://github.com/FilOzone/filecoin-services.git".to_string(), - tag: "v1.3.0".to_string(), - }, - pdp: None, - multicall3: Location::GitTag { - url: "https://github.com/mds1/multicall3.git".to_string(), - tag: "v3.1.0".to_string(), - }, + lotus: manifest.default_location("lotus")?, + curio: manifest.default_location("curio")?, + filecoin_services: manifest.default_location("filecoin-services")?, + pdp: manifest.default_optional_pdp_location()?, + multicall3: manifest.default_location("multicall3")?, approved_pdp_sp_count: 2, endorsed_pdp_sp_count: 1, active_pdp_sp_count: 2, + }) + } +} + +impl DependencyManifest { + fn component(&self, name: &str) -> Result<&DependencyComponent, String> { + self.dependencies + .get(name) + .ok_or_else(|| format!("Dependency manifest missing {name} dependency")) + } + + fn default_location(&self, name: &str) -> Result { + let component = self.component(name)?; + component.default.to_location(name, &component.repository) + } + + fn default_optional_pdp_location(&self) -> Result, String> { + let component = self.component("pdp")?; + match &component.default { + DependencySelection::Bundled { bundle, path } => { + parse_bundle_ref(bundle)?; + if path.is_empty() { + return Err("pdp default bundled path must not be empty".into()); + } + Ok(None) + } + selection => selection + .to_location("pdp", &component.repository) + .map(Some), + } + } +} + +impl DependencySelection { + fn to_location(&self, name: &str, repository: &str) -> Result { + match self { + DependencySelection::GitCommit { commit } => Ok(Location::GitCommit { + url: repository.to_string(), + commit: commit.clone(), + }), + DependencySelection::GitTag { tag } => Ok(Location::GitTag { + url: repository.to_string(), + tag: tag.clone(), + }), + DependencySelection::GitBranch { branch } => Ok(Location::GitBranch { + url: repository.to_string(), + branch: branch.clone(), + }), + DependencySelection::Bundled { .. } => Err(format!( + "{name} default cannot use bundled as a standalone location" + )), + DependencySelection::NpmVersion { version } => Err(format!( + "{name} default npm_version {version} cannot be used as a foc-devnet location" + )), } } } +fn parse_bundle_ref(bundle: &str) -> Result<(&str, &str), String> { + let (dependency, selection) = bundle + .split_once('@') + .ok_or_else(|| "bundled dependency must use '@'".to_string())?; + if dependency.is_empty() || selection.is_empty() || selection.contains('@') { + return Err("bundled dependency must use '@'".into()); + } + Ok((dependency, selection)) +} + impl Config { /// Validate configuration values. /// @@ -494,4 +587,144 @@ mod tests { assert!(serialized.contains("pdp")); assert!(serialized.contains("https://github.com/FilOzone/pdp.git")); } + + #[test] + fn default_config_reads_dependency_locations_from_manifest() { + let config = Config::default(); + + assert!(matches!( + config.lotus, + Location::GitTag { ref tag, .. } if tag == "v1.36.1" + )); + assert!(matches!( + config.curio, + Location::GitTag { ref tag, .. } if tag == "v1.28.2" + )); + assert!(matches!( + config.filecoin_services, + Location::GitTag { ref tag, .. } if tag == "v1.3.0" + )); + assert!(matches!( + config.multicall3, + Location::GitTag { ref tag, .. } if tag == "v3.1.0" + )); + assert!(config.pdp.is_none()); + } + + #[test] + fn manifest_parser_rejects_wrong_schema_version() { + let manifest = manifest_fixture( + 1, + r#" +[dependencies.pdp] +repository = "https://example.test/pdp.git" + +[dependencies.pdp.default] +strategy = "bundled" +bundle = "filecoin-services@default" +path = "service_contracts/lib/pdp" +"#, + ); + + let error = Config::default_from_dependency_manifest(&manifest).unwrap_err(); + + assert!(error.contains("schema_version must be 2")); + } + + #[test] + fn manifest_parser_rejects_missing_dependency_component() { + let manifest = r#" +schema_version = 2 + +[dependencies.lotus] +repository = "https://example.test/lotus.git" + +[dependencies.lotus.default] +strategy = "git_tag" +tag = "v1" + +[dependencies.curio] +repository = "https://example.test/curio.git" + +[dependencies.curio.default] +strategy = "git_tag" +tag = "v1" + +[dependencies.filecoin-services] +repository = "https://example.test/services.git" + +[dependencies.filecoin-services.default] +strategy = "git_tag" +tag = "v1" + +[dependencies.pdp] +repository = "https://example.test/pdp.git" + +[dependencies.pdp.default] +strategy = "bundled" +bundle = "filecoin-services@default" +path = "service_contracts/lib/pdp" +"#; + + let error = Config::default_from_dependency_manifest(manifest).unwrap_err(); + + assert!(error.contains("missing multicall3 dependency")); + } + + #[test] + fn manifest_parser_rejects_invalid_bundled_default() { + let manifest = manifest_fixture( + 2, + r#" +[dependencies.pdp] +repository = "https://example.test/pdp.git" + +[dependencies.pdp.default] +strategy = "bundled" +bundle = "filecoin-services" +path = "service_contracts/lib/pdp" +"#, + ); + + let error = Config::default_from_dependency_manifest(&manifest).unwrap_err(); + + assert!(error.contains("@")); + } + + fn manifest_fixture(schema_version: u8, pdp_component: &str) -> String { + format!( + r#" +schema_version = {schema_version} + +[dependencies.lotus] +repository = "https://example.test/lotus.git" + +[dependencies.lotus.default] +strategy = "git_tag" +tag = "v1" + +[dependencies.curio] +repository = "https://example.test/curio.git" + +[dependencies.curio.default] +strategy = "git_tag" +tag = "v1" + +[dependencies.filecoin-services] +repository = "https://example.test/services.git" + +[dependencies.filecoin-services.default] +strategy = "git_tag" +tag = "v1" + +[dependencies.multicall3] +repository = "https://example.test/multicall3.git" + +[dependencies.multicall3.default] +strategy = "git_tag" +tag = "v1" +{pdp_component} +"# + ) + } }