Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion packages/tangle-cli/src/tangle_cli/pipeline_compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1432,7 +1432,7 @@ def _build_local_from_python_components(
"""Build the ``<stem>.components.yaml`` content for @task refs.

Returns an ordered map ``{fragment: {name?, local_from_python:
{image?, function, dependencies_from?, file}}}``, DEDUPED by FUNCTION
{image?, function, mode?, resolve_root?, dependencies_from?, file}}}``, DEDUPED by FUNCTION
(the fragment = hyphenated function name). The SAME @task function
called from multiple task sites collapses to one entry; TWO DISTINCT
@task functions defined in ONE file each get their own entry (they
Expand Down Expand Up @@ -1496,6 +1496,16 @@ def _build_local_from_python_components(
# to the file stem and extracts the wrong symbol.
assert ref._task_function_name is not None
local_from_python["function"] = ref._task_function_name
if ref._task_mode is not None:
local_from_python["mode"] = ref._task_mode
if ref._task_resolve_root is not None:
resolve_root = ref._task_resolve_root
if not resolve_root.exists():
raise CompileError(
f"@task resolve_root is unreachable: {resolve_root}. "
"Point resolve_root at an existing directory or drop it."
)
local_from_python["resolve_root"] = _relpath_posix(resolve_root, components_yaml_dir)
if ref._task_dependencies_from is not None:
deps = ref._task_dependencies_from
if not deps.exists():
Expand Down
6 changes: 6 additions & 0 deletions packages/tangle-cli/src/tangle_cli/python_pipeline/ref.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ class CallableRef:
_task_function_name: str | None = None
_task_image: str | None = None
_task_dependencies_from: Path | None = None
_task_mode: str | None = None
_task_resolve_root: Path | None = None
_task_custom_annotations: dict[str, str] | None = None
# ``@registered`` metadata. ``None`` for ``ref()``/``@task`` refs;
# populated by the ``@registered`` decorator. Drives the compile-time
Expand Down Expand Up @@ -154,6 +156,8 @@ def materialize(self, output_path: Path | None = None) -> Path:
function_name=self._task_function_name,
dependencies_from=deps,
custom_annotations=self._task_custom_annotations,
mode=self._task_mode or "inline",
resolve_root=self._task_resolve_root,
)
else:
generator.regenerate_yaml(
Expand All @@ -162,6 +166,8 @@ def materialize(self, output_path: Path | None = None) -> Path:
function_name=self._task_function_name,
image=self._task_image,
dependencies_from=self._task_dependencies_from,
mode=self._task_mode or "inline",
resolve_root=self._task_resolve_root,
)
return output_path

Expand Down
30 changes: 26 additions & 4 deletions packages/tangle-cli/src/tangle_cli/python_pipeline/task.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""``@task`` decorator.

The decorator captures metadata about a Python function (source path,
function name, container image, dependencies, custom name, annotations)
function name, container image, dependencies, generation mode, resolve root, custom name, annotations)
and returns a :class:`CallableRef` with NO componentRef URL set.

At compile time the driver collects every ``@task`` ref that gets called
Expand Down Expand Up @@ -66,6 +66,8 @@ def task(
env: TaskEnv | None = None,
image: str | None = None,
dependencies_from: str | Path | None = None,
mode: str | None = None,
resolve_root: str | Path | None = None,
annotations: dict[str, Any] | None = None,
) -> Callable[[Callable[..., Any]], CallableRef]:
"""Decorator: turn a Python function into a Tangle component ref.
Expand Down Expand Up @@ -99,6 +101,14 @@ def task(
file when given as a string. Emitted into
``components.yaml#local_from_python.dependencies_from``.
Overrides ``env.dependencies_from`` when both are given.
mode: Optional local-from-python generation mode. ``None``
preserves the hydrator default (currently ``inline``).
Use ``"bundle"`` to ask hydrate-time codegen to embed
first-party imports using the existing module bundler.
resolve_root: Optional module resolution root for bundle mode.
Relative strings are resolved relative to the task source
file, then emitted into
``components.yaml#local_from_python.resolve_root``.
annotations: Extra annotations to merge into the emitted
component's ``metadata.annotations`` block.

Expand Down Expand Up @@ -143,6 +153,9 @@ def task(
# the normalisation below unchanged; an explicit relative string is
# still resolved relative to the @task source file.
raw_dependencies_from = effective_deps_raw
raw_resolve_root = resolve_root
if mode is not None and mode not in {"inline", "bundle"}:
raise ValueError("@task(mode=...) must be 'inline', 'bundle', or None")

def decorator(fn: Callable[..., Any]) -> CallableRef:
# Capture the absolute path of the source file the user wrote
Expand All @@ -161,9 +174,8 @@ def decorator(fn: Callable[..., Any]) -> CallableRef:

function_name = fn.__name__

# Resolve dependencies_from relative to the source file when
# the user gave a string. Absolute paths and explicit Path
# objects pass through unchanged.
# Resolve local paths relative to the source file when the user
# gave a relative value. Absolute paths pass through unchanged.
deps_path: Path | None
if raw_dependencies_from is None:
deps_path = None
Expand All @@ -172,6 +184,14 @@ def decorator(fn: Callable[..., Any]) -> CallableRef:
if not deps_path.is_absolute():
deps_path = (source_path.parent / deps_path).resolve()

resolve_root_path: Path | None
if raw_resolve_root is None:
resolve_root_path = None
else:
resolve_root_path = Path(raw_resolve_root)
if not resolve_root_path.is_absolute():
resolve_root_path = (source_path.parent / resolve_root_path).resolve()

# No URL set at decoration time -- the compile driver rewrites
# componentRef.url for @task-derived refs to
# ``resolve://./<out_stem>.components.yaml#<fragment>`` after
Expand All @@ -183,6 +203,8 @@ def decorator(fn: Callable[..., Any]) -> CallableRef:
_task_function_name=function_name,
_task_image=effective_image,
_task_dependencies_from=deps_path,
_task_mode=mode,
_task_resolve_root=resolve_root_path,
_task_custom_annotations=dict(annotations) if annotations else None,
)

Expand Down
31 changes: 31 additions & 0 deletions tests/test_pipeline_compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,37 @@ def test_compile_task_decorator_emits_sidecar(tmp_path):
validate_dehydrated_data(yaml.safe_load(out.read_text()))


def test_compile_task_decorator_emits_bundle_mode_and_resolve_root(tmp_path):
"""@task(mode="bundle") is carried into the auto-emitted sidecar."""
project = tmp_path / "project"
src = project / "src"
pipeline_path = src / "pipeline.py"
src.mkdir(parents=True)
(src / "helpers.py").write_text("MESSAGE = 'hello'\n", encoding="utf-8")
pipeline_path.write_text(
"from tangle_cli.python_pipeline import Out, pipeline, task\n"
"from helpers import MESSAGE\n\n"
"@task(image='python:3.12', mode='bundle', resolve_root='.')\n"
"def bundled_task() -> str:\n"
" return MESSAGE\n\n"
"@pipeline('Bundle Pipeline')\n"
"def bundle_pipeline() -> Out[str]:\n"
" bundled = bundled_task()\n"
" return bundled\n",
encoding="utf-8",
)

out = project / "compiled.yaml"
result = compile_pipeline(pipeline_path, out)

sidecar = yaml.safe_load(result.components_path.read_text())
local_from_python = sidecar["bundled-task"]["local_from_python"]
assert local_from_python["mode"] == "bundle"
assert local_from_python["resolve_root"] == "./src"
assert local_from_python["file"] == "./src/pipeline.py"
assert local_from_python["function"] == "bundled_task"


# ---------------------------------------------------------------------------
# Runnable argument-value emission (raw string constant / graphInput /
# taskOutput). Dispatch is on the VALUE's type, never the argument KEY.
Expand Down
41 changes: 41 additions & 0 deletions tests/test_pipelines_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -958,12 +958,15 @@ def _write_local_from_python_pipeline(
project_dir: Path,
python_file: str,
*,
mode: str | None = None,
resolve_root: str | None = None,
) -> Path:
gen_config = {
"file": python_file,
"output_folder": "./generated",
}
if mode is not None:
gen_config["mode"] = mode
if resolve_root is not None:
gen_config["resolve_root"] = resolve_root
_write_pipeline(
Expand Down Expand Up @@ -1020,6 +1023,44 @@ def fake_regenerate_yaml(**kwargs):
assert regenerated == [python_file.resolve()]


def test_pipelines_hydrate_local_from_python_forwards_bundle_mode_and_resolve_root(
monkeypatch,
tmp_path: Path,
):
from tangle_cli import pipeline_hydrator as hydrator_module
from tangle_cli.pipelines import hydrate_pipeline_file

project_dir = tmp_path / "project"
project_dir.mkdir()
src_dir = project_dir / "src"
src_dir.mkdir()
python_file = src_dir / "component.py"
python_file.write_text("# trusted project component\n", encoding="utf-8")
pipeline_path = _write_local_from_python_pipeline(
project_dir,
"./src/component.py",
mode="bundle",
resolve_root="./src",
)
calls: list[dict[str, object]] = []

def fake_regenerate_yaml(**kwargs):
calls.append(kwargs)
kwargs["output_path"].write_text(
"name: Generated Component\nimplementation:\n container:\n image: busybox\n",
encoding="utf-8",
)
return True

monkeypatch.setattr(hydrator_module, "regenerate_yaml", fake_regenerate_yaml)

hydrate_pipeline_file(pipeline_path)

assert calls[0]["python_file"] == python_file.resolve()
assert calls[0]["mode"] == "bundle"
assert calls[0]["resolve_root"] == src_dir.resolve()


def test_pipelines_hydrate_local_from_python_refuses_untrusted_absolute_path(
monkeypatch,
tmp_path: Path,
Expand Down
Loading