From b8c03b628f505c6d45a87836bfe859f398eff011 Mon Sep 17 00:00:00 2001 From: Hugo Pompougnac Date: Fri, 4 Sep 2026 13:51:53 +0200 Subject: [PATCH 1/5] mlir: encapsulate the XTC extensions to MLIR bindings --- .../backends/mlir/MlirBindingsExtensions.py | 43 +++++++++++++++++++ src/xtc/backends/mlir/MlirCompilerPasses.py | 9 ++-- 2 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 src/xtc/backends/mlir/MlirBindingsExtensions.py diff --git a/src/xtc/backends/mlir/MlirBindingsExtensions.py b/src/xtc/backends/mlir/MlirBindingsExtensions.py new file mode 100644 index 00000000..ca4fb6cc --- /dev/null +++ b/src/xtc/backends/mlir/MlirBindingsExtensions.py @@ -0,0 +1,43 @@ +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024-2026 The XTC Project Authors +# +"""Optional XTC extensions to the MLIR Python bindings.""" + +from __future__ import annotations + +import importlib +import logging + +logger = logging.getLogger(__name__) + +# Optional XTC extensions to the MLIR Python bindings: the module to import +# mapped to the pass-pipeline entries it contributes. +_EXTENSIONS: dict[str, tuple[str, ...]] = { + "mlir.xtc_transform": ("func.func(reduce-extract-slices)",), +} + +# Reverse map from a contributed pass to its providing module, so a pass can be +# gated without the caller naming its extension. +_PASS_OWNER: dict[str, str] = {} +for _module, _module_passes in _EXTENSIONS.items(): + for _pass in _module_passes: + assert _pass not in _PASS_OWNER, f"pass {_pass!r} declared by two extensions" + _PASS_OWNER[_pass] = _module + +# Extension modules that imported successfully, resolved once at import time. +_loaded: set[str] = set() +for _module in _EXTENSIONS: + try: + importlib.import_module(_module) + _loaded.add(_module) + except ImportError as _exc: + logger.debug("MLIR binding extension %r unavailable: %s", _module, _exc) + + +def passes(pass_names: list[str]) -> list[str]: + """Return the subset of ``pass_names`` whose extension is available.""" + for pass_name in pass_names: + if pass_name not in _PASS_OWNER: + raise KeyError(f"unknown extension pass: {pass_name!r}") + return [p for p in pass_names if _PASS_OWNER[p] in _loaded] diff --git a/src/xtc/backends/mlir/MlirCompilerPasses.py b/src/xtc/backends/mlir/MlirCompilerPasses.py index 20cee233..ae4ed40f 100644 --- a/src/xtc/backends/mlir/MlirCompilerPasses.py +++ b/src/xtc/backends/mlir/MlirCompilerPasses.py @@ -33,7 +33,8 @@ ) from mlir.passmanager import PassManager from mlir.ir import Module -import mlir.xtc_transform + +import xtc.backends.mlir.MlirBindingsExtensions as binding_extensions # Import SDist if available try: @@ -779,7 +780,6 @@ def run(self, pass_names: list[str]) -> None: def apply_bufferization_passes(mlir_program: RawMlirProgram, mlir_install_dir: str): - assert mlir.xtc_transform bufferize_options = [ "bufferize-function-boundaries", "function-boundary-type-conversion=identity-layout-map", @@ -787,9 +787,8 @@ def apply_bufferization_passes(mlir_program: RawMlirProgram, mlir_install_dir: s ] MlirProgramApplyPasses(mlir_program).run( - [ - # xtc pass that folds extract slices to make smaller tensor.empty allocations - "func.func(reduce-extract-slices)", + binding_extensions.passes(["func.func(reduce-extract-slices)"]) + + [ "canonicalize", "cse", "eliminate-empty-tensors", # causes ops to write directly to out buffer From dfa88fc54cff04812110505873323c1721f1238f Mon Sep 17 00:00:00 2001 From: Hugo Pompougnac Date: Fri, 4 Sep 2026 13:52:41 +0200 Subject: [PATCH 2/5] mlir: discard enable-x86vector, useless --- src/xtc/backends/mlir/MlirTarget/MlirLLVMTarget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xtc/backends/mlir/MlirTarget/MlirLLVMTarget.py b/src/xtc/backends/mlir/MlirTarget/MlirLLVMTarget.py index 4ecc57c3..dd9ca109 100644 --- a/src/xtc/backends/mlir/MlirTarget/MlirLLVMTarget.py +++ b/src/xtc/backends/mlir/MlirTarget/MlirLLVMTarget.py @@ -123,7 +123,7 @@ def _lowering_pipeline(self) -> list[str]: "sccp", # Data flow to LLVM "convert-math-to-llvm", - "convert-vector-to-llvm{enable-x86vector=true}", + "convert-vector-to-llvm", "convert-index-to-llvm", "convert-arith-to-llvm", "convert-ub-to-llvm", From 9ad01c1c7a8601f606c62337a71430914befbc80 Mon Sep 17 00:00:00 2001 From: Hugo Pompougnac Date: Fri, 4 Sep 2026 13:53:19 +0200 Subject: [PATCH 3/5] explore: warmup tvm before the other backends --- src/xtc/search/explore.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/xtc/search/explore.py b/src/xtc/search/explore.py index e0cb9c57..f0b9be35 100644 --- a/src/xtc/search/explore.py +++ b/src/xtc/search/explore.py @@ -864,8 +864,20 @@ def list_optimizers(): for name in Optimizers.names(): print(f"{name}") + def _warmup_llvm_backends(self) -> None: + """Initialize TVM's LLVM codegen before any other backend touches LLVM.""" + if "tvm" not in self.config.backends: + return + try: + import tvm + + tvm.target.codegen.llvm_version_major() + except Exception as exc: # best-effort: never fail the run on warmup + logger.debug("TVM LLVM warmup skipped: %s", exc) + def run(self) -> list[Sequence]: args = self.config + self._warmup_llvm_backends() if args.seed >= 0: np.random.seed(args.seed) random.seed(args.seed) From 67be1de2ca593733251895e60f4f2f64f9addbc5 Mon Sep 17 00:00:00 2001 From: Hugo Pompougnac Date: Fri, 4 Sep 2026 13:53:41 +0200 Subject: [PATCH 4/5] readme: describe how to use a local LLVM checkout --- Makefile | 4 ++-- README.md | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 17cbc513..81f00bf2 100644 --- a/Makefile +++ b/Makefile @@ -97,10 +97,10 @@ pages: $(MAKE) -C docs/site site agents: - scripts/llms/init_agents.py agents README.md "Links" "AI assistants" > AGENTS.md + scripts/llms/init_agents.py agents README.md "Links" "local LLVM" "AI assistants" > AGENTS.md claude: - scripts/llms/init_agents.py claude README.md "Links" "AI assistants" > CLAUDE.md + scripts/llms/init_agents.py claude README.md "Links" "local LLVM" "AI assistants" > CLAUDE.md run-tutorial: marimo run docs/tutorials/xtc_101.py diff --git a/README.md b/README.md index 8f49f91b..6936371a 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,38 @@ Available extensions are: - `[default]`: mlir + tvm - `[dev]`: default + test +### Using a local LLVM / MLIR build + +By default XTC uses the LLVM/MLIR toolchain shipped in its Python wheels. To +use a local LLVM checkout instead (e.g. to test compiler changes), point XTC +at your build directory (`$LLVM_BUILD` below). Two levels are possible. + +**Binaries only.** Override `opt`, `llc`, `mlir-opt` and `mlir-translate` +while keeping the wheel's Python bindings — the simplest option, e.g. when +your changes live in the LLVM middle-end / back-end: + +```bash +export XTC_LLVM_PREFIX=$LLVM_BUILD # providing bin/opt and bin/llc +export XTC_MLIR_PREFIX=$LLVM_BUILD # providing bin/mlir-opt and bin/mlir-translate +``` + +**Binaries and Python bindings.** The MLIR bindings are a native extension, +so the interpreter must match the Python version they were built for (check +the ABI tag under +`$LLVM_BUILD/tools/mlir/python_packages/mlir_core/mlir/_mlir_libs`). +Create a matching venv if, install XTC without the MLIR wheels, build the +runtime support libraries, then prepend the bindings to `PYTHONPATH`: + +```bash +uv venv -p 3.14 .venv-local && source .venv-local/bin/activate +uv pip install -e '.[tvm]' + +export PYTHONPATH=$LLVM_BUILD/tools/mlir/python_packages/mlir_core:$PYTHONPATH +export XTC_MLIR_PREFIX=$LLVM_BUILD +export XTC_LLVM_PREFIX=$LLVM_BUILD +export XTC_MLIR_TARGET=llvmir +``` + ### Code quality Code quality requirements: From f6ea98fe7a9da46c031b7103ce65c5a0ace3f260 Mon Sep 17 00:00:00 2001 From: Hugo Pompougnac Date: Sun, 6 Sep 2026 13:13:10 +0200 Subject: [PATCH 5/5] system: more generic binutils --- src/xtc/utils/host_tools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/xtc/utils/host_tools.py b/src/xtc/utils/host_tools.py index 3a54698d..4662fb12 100644 --- a/src/xtc/utils/host_tools.py +++ b/src/xtc/utils/host_tools.py @@ -45,6 +45,9 @@ def binutils_prefix(arch: str = "") -> str: triple = target_triple(arch) if not triple: return "" + if platform.system() == "Linux" and target_arch(arch) == platform.machine(): + # Native target: the host binutils handle it, no cross prefix needed + return "" if platform.system() == "Darwin" and triple == "aarch64-linux-gnu": # On darwin cross aarch64 binutils from aarch64-elf-binutils triple = "aarch64-elf"