From 0fa2797b04fbd71750d1c0810ec48c692ec8da5a Mon Sep 17 00:00:00 2001 From: Volv G Date: Tue, 21 Jul 2026 16:56:29 -0400 Subject: [PATCH] Add task image IDs and compile-time image overrides Add an OSS-neutral image-id seam for Python-authored @task components: - accept @task(image_id="...") alongside the existing image= option - keep image= fully backward-compatible and higher precedence - add an empty IMAGE_IDS registry with register/get/resolve helpers for downstream defaults - plumb repeatable `tangle sdk pipelines compile --image ID=REF` overrides through the CLI facade and compiler - resolve local_from_python images by precedence: image=, override, registered default; error clearly when an image_id is unresolved - cover registry, sidecar emission, override resolution, precedence, CLI help, compat imports, and packaging - bump tangle-cli to 0.1.6 Tests: - uv lock --check - uv run pytest -q (795 passed, 4 warnings) Assisted-By: devx/b49d7847-4e8a-45eb-8304-dacd6e78458e --- .../tangle-cli/src/tangle_cli/__init__.py | 2 +- .../src/tangle_cli/pipeline_compiler.py | 63 ++++++++++- .../tangle-cli/src/tangle_cli/pipelines.py | 2 + .../src/tangle_cli/pipelines_cli.py | 24 ++++ .../python_pipeline/compiler_context.py | 1 + .../src/tangle_cli/python_pipeline/ref.py | 1 + .../src/tangle_cli/python_pipeline/task.py | 12 +- pyproject.toml | 2 +- tests/test_component_from_func.py | 16 +++ tests/test_packaging.py | 2 +- tests/test_pipeline_compiler.py | 105 ++++++++++++++++++ tests/test_pipelines_cli.py | 10 ++ tests/test_tangle_deploy_compat_imports.py | 24 ++++ uv.lock | 2 +- 14 files changed, 259 insertions(+), 7 deletions(-) diff --git a/packages/tangle-cli/src/tangle_cli/__init__.py b/packages/tangle-cli/src/tangle_cli/__init__.py index e04a2a4..1699ed3 100644 --- a/packages/tangle-cli/src/tangle_cli/__init__.py +++ b/packages/tangle-cli/src/tangle_cli/__init__.py @@ -14,6 +14,6 @@ try: __version__ = metadata_version("tangle-cli") except PackageNotFoundError: - __version__ = "0.1.5" + __version__ = "0.1.6" __all__ = ["TangleDynamicDiscoveryClient", "__version__"] diff --git a/packages/tangle-cli/src/tangle_cli/pipeline_compiler.py b/packages/tangle-cli/src/tangle_cli/pipeline_compiler.py index e65c9ca..44e914a 100644 --- a/packages/tangle-cli/src/tangle_cli/pipeline_compiler.py +++ b/packages/tangle-cli/src/tangle_cli/pipeline_compiler.py @@ -72,6 +72,40 @@ from .utils import dump_yaml +IMAGE_IDS: dict[str, str] = {} +"""Logical image-id registry used by ``@task(image_id=...)``. + +OSS ships this empty. Downstream distributions may register environment-specific +image defaults without hard-coding their image literals into the open-source +package; CLI ``--image ID=REF`` overrides still win over registered defaults. +""" + + +def register_image_id(image_id: str, image: str) -> None: + """Register a default image reference for ``@task(image_id=...)``.""" + if not image_id: + raise ValueError("image_id must be a non-empty string") + if not image: + raise ValueError("image must be a non-empty string") + IMAGE_IDS[image_id] = image + + +def get_image_id(image_id: str) -> str | None: + """Return the registered default image for ``image_id``, if any.""" + return IMAGE_IDS.get(image_id) + + +def resolve_image_id( + image_id: str, + image_overrides: Mapping[str, str] | None = None, +) -> str | None: + """Resolve ``image_id`` using compile-time overrides, then registered defaults.""" + overrides = image_overrides or {} + if image_id in overrides: + return overrides[image_id] + return get_image_id(image_id) + + @dataclass class CompileResult: """Outcome of a :func:`compile_pipeline` call. @@ -102,6 +136,7 @@ def compile_pipeline( *, pipeline_name: str | None = None, emit_components_sidecar: bool = True, + image_overrides: Mapping[str, str] | None = None, ) -> CompileResult: """Compile ``script`` to a single pipeline YAML at ``output``. @@ -132,6 +167,9 @@ def compile_pipeline( compilation fails with a :class:`CompileError` (a valid dehydrated pipeline cannot be produced without the sidecar). Pipelines that use only ``ref(url=...)`` are unaffected. + image_overrides: Compile-time image-id overrides from ``--image + ID=REF``. They apply only to ``@task(image_id=...)`` refs that do + not also set an explicit ``image=``. Returns: A :class:`CompileResult`. ``components_path`` is the sidecar path @@ -143,6 +181,7 @@ def compile_pipeline( unreachable ``@task`` source/dependency files). """ overrides = dict(overrides or {}) + image_overrides = dict(image_overrides or {}) # 1. Validate the script path. script_path = Path(script).resolve() @@ -197,6 +236,7 @@ def compile_pipeline( root_overrides=overrides, emit_components_sidecar=emit_components_sidecar, source_dirs=purge_dirs, + image_overrides=image_overrides, ) # 5. Compile the root (and, recursively, all children) into in-memory @@ -436,7 +476,11 @@ def _compile_pipeline_fn( ) if task_refs: components_path = output_path.with_name(output_path.stem + ".components.yaml") - components_entries = _build_local_from_python_components(task_refs, components_yaml_dir=components_path.parent) + components_entries = _build_local_from_python_components( + task_refs, + components_yaml_dir=components_path.parent, + image_overrides=ctx.image_overrides, + ) _rewrite_task_componentref_urls( body_dict=body_dict, task_refs=task_refs, @@ -1427,7 +1471,10 @@ def _relocate_relative_local_url(url: str, source_dir: Path, sidecar_dir: Path) def _build_local_from_python_components( - task_refs: list[tuple[str, CallableRef]], *, components_yaml_dir: Path + task_refs: list[tuple[str, CallableRef]], + *, + components_yaml_dir: Path, + image_overrides: Mapping[str, str] | None = None, ) -> dict[str, Any]: """Build the ``.components.yaml`` content for @task refs. @@ -1492,6 +1539,16 @@ def _build_local_from_python_components( local_from_python: dict[str, Any] = {} if ref._task_image is not None: local_from_python["image"] = ref._task_image + elif ref._task_image_id is not None: + resolved_image = resolve_image_id(ref._task_image_id, image_overrides) + if resolved_image is None: + raise CompileError( + f"@task image_id={ref._task_image_id!r} on function " + f"{ref._task_function_name!r} did not resolve to an image. " + f"Pass --image {ref._task_image_id}=IMAGE to `tangle sdk pipelines compile`, " + f"or register a default with register_image_id({ref._task_image_id!r}, IMAGE)." + ) + local_from_python["image"] = resolved_image # Always pin the function name. Without it the hydrator defaults # to the file stem and extracts the wrong symbol. assert ref._task_function_name is not None @@ -2233,6 +2290,7 @@ def compile_file( overrides: Mapping[str, str] | None = None, pipeline_name: str | None = None, emit_components_sidecar: bool = True, + image_overrides: Mapping[str, str] | None = None, ) -> CompileResult: """Compile ``script`` to a single dehydrated pipeline YAML at ``output``. @@ -2251,6 +2309,7 @@ def compile_file( overrides, pipeline_name=pipeline_name, emit_components_sidecar=emit_components_sidecar, + image_overrides=image_overrides, ) self.log.info(f"wrote {result.pipeline_path}") if result.components_path is not None: diff --git a/packages/tangle-cli/src/tangle_cli/pipelines.py b/packages/tangle-cli/src/tangle_cli/pipelines.py index 15b64b5..b843bdc 100644 --- a/packages/tangle-cli/src/tangle_cli/pipelines.py +++ b/packages/tangle-cli/src/tangle_cli/pipelines.py @@ -258,6 +258,7 @@ def compile_pipeline_file( overrides: Mapping[str, str] | None = None, pipeline_name: str | None = None, emit_components_sidecar: bool = True, + image_overrides: Mapping[str, str] | None = None, logger: Any | None = None, ) -> CompileResult: """Compile a Python-authored pipeline to a dehydrated YAML bundle. @@ -284,6 +285,7 @@ def compile_pipeline_file( overrides=dict(overrides) if overrides else None, pipeline_name=pipeline_name, emit_components_sidecar=emit_components_sidecar, + image_overrides=dict(image_overrides) if image_overrides else None, ) except (CompileError, SchemaValidationError) as exc: raise PipelineValidationError(str(exc)) from exc diff --git a/packages/tangle-cli/src/tangle_cli/pipelines_cli.py b/packages/tangle-cli/src/tangle_cli/pipelines_cli.py index 6f5f22b..3a30e57 100644 --- a/packages/tangle-cli/src/tangle_cli/pipelines_cli.py +++ b/packages/tangle-cli/src/tangle_cli/pipelines_cli.py @@ -138,6 +138,18 @@ def _parse_overrides(values: list[str] | None) -> dict[str, str]: return parsed +def _parse_image_overrides(values: list[str] | None) -> dict[str, str]: + parsed: dict[str, str] = {} + for value in values or []: + if "=" not in value: + raise SystemExit("--image entries must use ID=REF syntax") + image_id, image_ref = value.split("=", 1) + if not image_id or not image_ref: + raise SystemExit("--image entries must use ID=REF syntax") + parsed[image_id] = image_ref + return parsed + + @app.command(name="hydrate") def pipelines_hydrate( pipeline_path: pathlib.Path, @@ -266,6 +278,17 @@ def pipelines_compile( negative_iterable=(), ), ] = None, + image: Annotated[ + list[str] | None, + Parameter( + name="--image", + help=( + "Compile-time image-id override as ID=REF for @task(image_id=ID). " + "Repeat for multiple IDs." + ), + negative_iterable=(), + ), + ] = None, log_type: LogTypeOption = "console", ) -> None: """Compile a Python-authored pipeline to a dehydrated YAML bundle.""" @@ -276,6 +299,7 @@ def pipelines_compile( pipeline_path, output, overrides=_parse_overrides(override), + image_overrides=_parse_image_overrides(image), pipeline_name=pipeline, logger=logger, ) diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/compiler_context.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/compiler_context.py index 6dd96cb..a54d270 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/compiler_context.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/compiler_context.py @@ -252,6 +252,7 @@ class CompileContext: subgraph_dir: Path root_overrides: dict[str, str] = field(default_factory=dict) emit_components_sidecar: bool = True + image_overrides: dict[str, str] = field(default_factory=dict) max_depth: int = 32 # Compiled CHILD artifacts keyed by compile key (Decision M dedup). # The root is NOT stored here; it is returned directly. diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py index c09102d..f7491ef 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py @@ -49,6 +49,7 @@ class CallableRef: _task_source_path: Path | None = None _task_function_name: str | None = None _task_image: str | None = None + _task_image_id: str | None = None _task_dependencies_from: Path | None = None _task_mode: str | None = None _task_resolve_root: Path | None = None diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/task.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/task.py index 36fdf04..25b0f7c 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/task.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/task.py @@ -65,6 +65,7 @@ def task( *, env: TaskEnv | None = None, image: str | None = None, + image_id: str | None = None, dependencies_from: str | Path | None = None, mode: str | None = None, resolve_root: str | Path | None = None, @@ -95,6 +96,10 @@ def task( written verbatim into the emitted ``components.yaml#local_from_python.image`` field. Overrides ``env.image`` when both are given. + image_id: Logical image identifier resolved at compile time via + registered defaults or ``tangle sdk pipelines compile --image + ID=REF`` overrides. Ignored when explicit ``image=`` supplies + an image; otherwise it also overrides ``env.image``. dependencies_from: Path to a ``pyproject.toml`` (or any file the hydrator understands) that declares pip dependencies. Resolved relative to the caller's source @@ -138,7 +143,11 @@ def task( # overrides the corresponding ``env`` field; otherwise the env value # (if any) is used. This mirrors YAML anchor semantics: start from the # declared-once defaults, override locally where needed. - effective_image = image if image is not None else (env.image if env else None) + effective_image = ( + image + if image is not None + else (None if image_id is not None else (env.image if env else None)) + ) effective_deps_raw = ( dependencies_from if dependencies_from is not None @@ -202,6 +211,7 @@ def decorator(fn: Callable[..., Any]) -> CallableRef: _task_source_path=source_path, _task_function_name=function_name, _task_image=effective_image, + _task_image_id=image_id, _task_dependencies_from=deps_path, _task_mode=mode, _task_resolve_root=resolve_root_path, diff --git a/pyproject.toml b/pyproject.toml index 58cb436..e494f5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tangle-cli" -version = "0.1.5" +version = "0.1.6" description = "CLI for Tangle, the open-source ML pipeline orchestration platform" readme = "README.md" authors = [ diff --git a/tests/test_component_from_func.py b/tests/test_component_from_func.py index 444b28e..259a911 100644 --- a/tests/test_component_from_func.py +++ b/tests/test_component_from_func.py @@ -50,6 +50,22 @@ # ``sys.modules``) is provided suite-wide by the autouse # ``downstream_authoring_surface`` fixture in ``tests/conftest.py``. +# ============================================================================ +# @task image-id authoring metadata +# ============================================================================ + + +def test_task_decorator_records_image_id_without_explicit_image(): + from tangle_cli.python_pipeline import task + + @task(image_id="eval-slim") + def uses_image_id() -> str: + return "ok" + + assert uses_image_id._task_image is None + assert uses_image_id._task_image_id == "eval-slim" + + # ============================================================================ # Type resolution tests # ============================================================================ diff --git a/tests/test_packaging.py b/tests/test_packaging.py index dd56d23..2bd255e 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -178,7 +178,7 @@ def test_tangle_cli_wheel_supports_expert_no_deps_import_path_without_tangle_api requires_dist = [line for line in metadata.splitlines() if line.startswith("Requires-Dist: ")] assert not any(name.startswith("tangle_api/") for name in names) assert "tangle_cli/openapi/openapi.json" not in names - assert "Version: 0.1.5" in metadata + assert "Version: 0.1.6" in metadata assert "Requires-Dist: tangle-api==0.1.1" in requires_dist assert not any("extra == 'native'" in line for line in requires_dist) assert "Provides-Extra: native" in metadata diff --git a/tests/test_pipeline_compiler.py b/tests/test_pipeline_compiler.py index e6cf0d8..622c5b6 100644 --- a/tests/test_pipeline_compiler.py +++ b/tests/test_pipeline_compiler.py @@ -16,10 +16,12 @@ from tangle_cli import cli from tangle_cli.pipeline_compiler import ( + IMAGE_IDS, ZONE_ROOT_MARKERS, CompileResult, PipelineCompiler, compile_pipeline, + register_image_id, ) from tangle_cli.pipelines import PipelineValidationError, compile_pipeline_file from tangle_cli.python_pipeline.errors import CompileError @@ -375,6 +377,85 @@ def test_compile_task_decorator_emits_bundle_mode_and_resolve_root(tmp_path): assert local_from_python["function"] == "bundled_task" +def _write_image_id_pipeline(project: Path, decorator: str) -> Path: + src = project / "src" + src.mkdir(parents=True) + pipeline_path = src / "pipeline.py" + pipeline_path.write_text( + "from tangle_cli.python_pipeline import Out, pipeline, task\n\n" + f"@task({decorator})\n" + "def image_task() -> str:\n" + " return 'ok'\n\n" + "@pipeline('Image Pipeline')\n" + "def image_pipeline() -> Out[str]:\n" + " result = image_task()\n" + " return result\n", + encoding="utf-8", + ) + return pipeline_path + + +def test_compile_task_image_id_uses_registered_default(tmp_path): + original = dict(IMAGE_IDS) + try: + IMAGE_IDS.clear() + register_image_id("eval-slim", "registry.example/eval-slim:latest") + pipeline_path = _write_image_id_pipeline(tmp_path / "project", "image_id='eval-slim'") + + result = compile_pipeline(pipeline_path, tmp_path / "project" / "compiled.yaml") + + sidecar = yaml.safe_load(result.components_path.read_text()) + local_from_python = sidecar["image-task"]["local_from_python"] + assert local_from_python["image"] == "registry.example/eval-slim:latest" + finally: + IMAGE_IDS.clear() + IMAGE_IDS.update(original) + + +def test_compile_task_image_id_uses_compile_time_override(tmp_path): + pipeline_path = _write_image_id_pipeline(tmp_path / "project", "image_id='eval-slim'") + + result = compile_pipeline( + pipeline_path, + tmp_path / "project" / "compiled.yaml", + image_overrides={"eval-slim": "registry.example/eval-slim@sha256:abc"}, + ) + + sidecar = yaml.safe_load(result.components_path.read_text()) + local_from_python = sidecar["image-task"]["local_from_python"] + assert local_from_python["image"] == "registry.example/eval-slim@sha256:abc" + + +def test_compile_task_explicit_image_precedes_image_id_override(tmp_path): + pipeline_path = _write_image_id_pipeline( + tmp_path / "project", + "image='python:3.12', image_id='eval-slim'", + ) + + result = compile_pipeline( + pipeline_path, + tmp_path / "project" / "compiled.yaml", + image_overrides={"eval-slim": "registry.example/eval-slim@sha256:abc"}, + ) + + sidecar = yaml.safe_load(result.components_path.read_text()) + local_from_python = sidecar["image-task"]["local_from_python"] + assert local_from_python["image"] == "python:3.12" + + +def test_compile_task_image_id_without_default_or_override_fails(tmp_path): + original = dict(IMAGE_IDS) + try: + IMAGE_IDS.clear() + pipeline_path = _write_image_id_pipeline(tmp_path / "project", "image_id='eval-slim'") + + with pytest.raises(CompileError, match="image_id='eval-slim'.*did not resolve"): + compile_pipeline(pipeline_path, tmp_path / "project" / "compiled.yaml") + finally: + IMAGE_IDS.clear() + IMAGE_IDS.update(original) + + # --------------------------------------------------------------------------- # Runnable argument-value emission (raw string constant / graphInput / # taskOutput). Dispatch is on the VALUE's type, never the argument KEY. @@ -523,6 +604,30 @@ def test_compile_cli_writes_output(tmp_path, capsys): assert "Compiled" in capsys.readouterr().out +def test_compile_cli_image_override_writes_resolved_image(tmp_path): + project = tmp_path / "project" + out = project / "compiled.yaml" + pipeline_path = _write_image_id_pipeline(project, "image_id='eval-slim'") + app = cli.build_app() + + run_app( + app, + [ + "sdk", + "pipelines", + "compile", + str(pipeline_path), + "-o", + str(out), + "--image", + "eval-slim=registry.example/eval-slim@sha256:abc", + ], + ) + + sidecar = yaml.safe_load(out.with_name("compiled.components.yaml").read_text()) + assert sidecar["image-task"]["local_from_python"]["image"] == "registry.example/eval-slim@sha256:abc" + + def test_compile_cli_help_exits_zero(capsys): app = cli.build_app() run_app(app, ["sdk", "pipelines", "compile", "--help"]) diff --git a/tests/test_pipelines_cli.py b/tests/test_pipelines_cli.py index 0e59c3d..efd78f8 100644 --- a/tests/test_pipelines_cli.py +++ b/tests/test_pipelines_cli.py @@ -78,6 +78,16 @@ def test_sdk_help_includes_pipelines(capsys): assert "secrets" in output +def test_pipelines_compile_help_includes_image_override(capsys): + app = cli.build_app() + + run_app(app, ["sdk", "pipelines", "compile", "--help"]) + + output = capsys.readouterr().out + assert "--image" in output + assert "ID=REF" in output + + def test_sdk_pipelines_help_lists_local_commands(capsys): app = cli.build_app() diff --git a/tests/test_tangle_deploy_compat_imports.py b/tests/test_tangle_deploy_compat_imports.py index 8ccf4af..bf45c95 100644 --- a/tests/test_tangle_deploy_compat_imports.py +++ b/tests/test_tangle_deploy_compat_imports.py @@ -220,10 +220,14 @@ def test_tangle_deploy_pipeline_compile_import_surface() -> None: validate_dehydrated_pipeline, ) from tangle_cli.pipeline_compiler import ( + IMAGE_IDS, CompileResult, PipelineCompiler, ZONE_ROOT_MARKERS, compile_pipeline, + get_image_id, + register_image_id, + resolve_image_id, ) from tangle_cli.handler import TangleCliHandler @@ -249,6 +253,26 @@ def test_tangle_deploy_pipeline_compile_import_surface() -> None: assert issubclass(PipelineCompiler, TangleCliHandler) assert callable(PipelineCompiler.compile_file) assert isinstance(ZONE_ROOT_MARKERS, list) + assert isinstance(IMAGE_IDS, dict) + assert callable(register_image_id) + assert callable(get_image_id) + assert callable(resolve_image_id) + + +def test_image_id_registry_is_empty_and_mutable_for_downstream() -> None: + from tangle_cli import pipeline_compiler as pc + + assert pc.IMAGE_IDS == {} + original = dict(pc.IMAGE_IDS) + try: + pc.register_image_id("eval-slim", "registry.example/eval-slim:latest") + assert pc.get_image_id("eval-slim") == "registry.example/eval-slim:latest" + assert pc.resolve_image_id("eval-slim", {"eval-slim": "override"}) == "override" + finally: + pc.IMAGE_IDS.clear() + pc.IMAGE_IDS.update(original) + assert pc.IMAGE_IDS == {} + def test_zone_root_markers_seam_is_empty_and_mutable_for_downstream() -> None: diff --git a/uv.lock b/uv.lock index b5740fd..00ae3fd 100644 --- a/uv.lock +++ b/uv.lock @@ -2083,7 +2083,7 @@ requires-dist = [{ name = "pydantic", specifier = ">=2.0" }] [[package]] name = "tangle-cli" -version = "0.1.5" +version = "0.1.6" source = { editable = "." } dependencies = [ { name = "cloud-pipelines" },