Expose the matching engine to Python via pybind11

.github/workflows/CI.yml

@@ -35,3 +35,28 @@ jobs:
 
       - name: Run tests
         run: ctest --test-dir build --output-on-failure
+
+  python-bindings:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install build tools
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y cmake ninja-build g++
+
+      # No FetchContent cache: with tests and benchmarks off, this configure
+      # clones nothing — pip supplies pybind11 to the isolated build environment.
+      - name: Build and install the extension
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install '.[test]' -v
+
+      - name: Run Python tests
+        run: python -m pytest tests/python -v

.gitignore

@@ -2,3 +2,9 @@
 .claude
 build*/
 data/
+
+# Python
+.venv/
+__pycache__/
+*.egg-info/
+.pytest_cache/

CMakeLists.txt

@@ -22,7 +22,20 @@ endif()
 
 # Compile commands json for clangd / IDE integration
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-
+
+# -----------------------------------------------------------------------------
+# Build options. Every default reproduces the historical build exactly, so
+#   cmake -S . -B build && cmake --build build && ctest --test-dir build
+# is unaffected by their existence. The wheel build (scikit-build-core, see
+# pyproject.toml) turns tests/bench/apps off and python on, so `pip install .`
+# never clones googletest or benchmark.
+# -----------------------------------------------------------------------------
+option(ME_BUILD_TESTS  "Build the GoogleTest unit tests"                  ON)
+option(ME_BUILD_BENCH  "Build the Google Benchmark suites"                ON)
+option(ME_BUILD_APPS   "Build the me_main demo executable"                ON)
+option(ME_BUILD_PYTHON "Build the pybind11 extension module"              OFF)
+option(ME_NATIVE_ARCH  "Tune Release builds for this CPU (-march=native)" ON)
+
 # -----------------------------------------------------------------------------
 # Warning flags applied via an interface target (modern pattern)
 # Link any target against `project_warnings` to inherit these.
@@ -37,9 +50,13 @@ target_compile_options(project_warnings INTERFACE
 )
 
 # Performance flags for Release (Debug already has -O0 -g by default)
+# -march=native is behind ME_NATIVE_ARCH: a binary built with it dies with SIGILL
+# on any machine older than the one that built it, which is fine locally and fatal
+# for a redistributable wheel. With the option ON (the default) this expands
+# exactly as it always did.
 add_library(project_options INTERFACE)
 target_compile_options(project_options INTERFACE
-    $<$<CONFIG:Release>:-O3 -march=native -DNDEBUG>
+    $<$<CONFIG:Release>:-O3 $<$<BOOL:${ME_NATIVE_ARCH}>:-march=native> -DNDEBUG>
     $<$<CONFIG:Debug>:-O0 -g3 -fno-omit-frame-pointer>
 )
 
@@ -51,24 +68,32 @@ include(FetchContent)
 # The logger runs on its own thread and LockQueue uses std::mutex/condition_variable.
 find_package(Threads REQUIRED)
 
+# Each dependency is fetched only when the thing that needs it is being built, so
+# a Python-only configure (pip install .) clones neither. benchmark does not need
+# googletest here because BENCHMARK_ENABLE_TESTING is forced OFF below.
+
 # GoogleTest
-FetchContent_Declare(
-    googletest
-    GIT_REPOSITORY https://github.com/google/googletest.git
-    GIT_TAG        v1.14.0
-)
-
+if(ME_BUILD_TESTS)
+    FetchContent_Declare(
+        googletest
+        GIT_REPOSITORY https://github.com/google/googletest.git
+        GIT_TAG        v1.14.0
+    )
+    FetchContent_MakeAvailable(googletest)
+endif()
+
 # Google Benchmark
-set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE)
-set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "" FORCE)
-FetchContent_Declare(
-    benchmark
-    GIT_REPOSITORY https://github.com/google/benchmark.git
-    GIT_TAG        v1.8.3
-)
-
-FetchContent_MakeAvailable(googletest benchmark)
-
+if(ME_BUILD_BENCH)
+    set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE)
+    set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "" FORCE)
+    FetchContent_Declare(
+        benchmark
+        GIT_REPOSITORY https://github.com/google/benchmark.git
+        GIT_TAG        v1.8.3
+    )
+    FetchContent_MakeAvailable(benchmark)
+endif()
+
 # -----------------------------------------------------------------------------
 # Core library
 # -----------------------------------------------------------------------------
@@ -86,19 +111,29 @@ target_include_directories(me_core PUBLIC
 )
 target_link_libraries(me_core PRIVATE project_warnings project_options)
 
+# The Python extension is a shared object, so every object linked into it must be
+# position independent; without this the module fails to link on x86-64 with
+# R_X86_64_32S relocation errors. Set unconditionally rather than behind
+# ME_BUILD_PYTHON so there is one build of me_core instead of a PIC and a non-PIC
+# variant that can drift apart — the hot path has no globals, so the extra
+# indirection does not show up in bench/.
+set_target_properties(me_core PROPERTIES POSITION_INDEPENDENT_CODE ON)
+
 # -----------------------------------------------------------------------------
 # Main executable
 # -----------------------------------------------------------------------------
-add_executable(me_main src/main.cpp)
-target_link_libraries(me_main PRIVATE
-    me_core
-    me_ds_lock          # LockQueue
-    me_event_handler    # Logger
-    Threads::Threads
-    project_warnings
-    project_options
-)
-
+if(ME_BUILD_APPS)
+    add_executable(me_main src/main.cpp)
+    target_link_libraries(me_main PRIVATE
+        me_core
+        me_ds_lock          # LockQueue
+        me_event_handler    # Logger
+        Threads::Threads
+        project_warnings
+        project_options
+    )
+endif()
+
 # -----------------------------------------------------------------------------
 # Lock-based data structures (practice) — header-only.
 # Headers live in include/data_structures/; there is no .cpp, so this is an
@@ -119,10 +154,24 @@ target_link_libraries(me_event_handler INTERFACE me_core Threads::Threads)
 # -----------------------------------------------------------------------------
 # Tests
 # -----------------------------------------------------------------------------
-enable_testing()
-add_subdirectory(tests)
-
+if(ME_BUILD_TESTS)
+    enable_testing()
+    add_subdirectory(tests)
+endif()
+
 # -----------------------------------------------------------------------------
 # Benchmarks
 # -----------------------------------------------------------------------------
-add_subdirectory(bench)
+if(ME_BUILD_BENCH)
+    add_subdirectory(bench)
+endif()
+
+# -----------------------------------------------------------------------------
+# Python bindings (opt-in; scikit-build-core sets ME_BUILD_PYTHON=ON)
+# Last in the file on purpose: the module links me_ds_lock and me_event_handler,
+# and unlike me_main a subdirectory cannot forward-reference targets that do not
+# exist yet.
+# -----------------------------------------------------------------------------
+if(ME_BUILD_PYTHON)
+    add_subdirectory(src/python)
+endif()

pyproject.toml

@@ -0,0 +1,40 @@
+[build-system]
+requires = ["scikit-build-core>=0.10", "pybind11>=2.12,<4"]
+build-backend = "scikit_build_core.build"
+
+[project]
+name = "matching-engine"
+version = "0.1.0"                # keep in step with project(VERSION) in CMakeLists.txt
+description = "Python bindings for a single-symbol C++ matching engine"
+requires-python = ">=3.9"
+classifiers = [
+    "Programming Language :: C++",
+    "Programming Language :: Python :: 3",
+    "Topic :: Office/Business :: Financial :: Investment",
+]
+
+[project.optional-dependencies]
+test = ["pytest>=7"]
+
+[tool.scikit-build]
+minimum-version = "build-system.requires"
+cmake.version = ">=3.20"
+cmake.build-type = "Release"
+# Under build*/, which .gitignore already covers. The wheel tag keeps it clear of
+# a developer's own build/ tree and makes rebuilds incremental.
+build-dir = "build/wheel-{wheel_tag}"
+wheel.packages = ["python/matching_engine"]
+sdist.exclude = ["logs/", ".vscode/", ".claude/"]
+
+# Only the extension is wanted here: the C++ tests and benchmarks would drag
+# googletest and Google Benchmark in over the network for no reason.
+# ME_NATIVE_ARCH is left at its default ON, which is right for `pip install .` on
+# the machine that will run it. A redistributable wheel wants -DME_NATIVE_ARCH=OFF.
+[tool.scikit-build.cmake.define]
+ME_BUILD_PYTHON = "ON"
+ME_BUILD_TESTS = "OFF"
+ME_BUILD_BENCH = "OFF"
+ME_BUILD_APPS = "OFF"
+
+[tool.pytest.ini_options]
+testpaths = ["tests/python"]

python/matching_engine/__init__.py

@@ -0,0 +1,40 @@
+"""A single-symbol matching engine, implemented in C++.
+
+Submit orders to a :class:`MatchingEngine` and it maintains a price-time priority
+book, matching crossing orders on arrival and applying pre-trade risk checks:
+
+    >>> from matching_engine import MatchingEngine, OrderSide
+    >>> with MatchingEngine("logs/session.log") as engine:
+    ...     buy = engine.add_order(price=1005, quantity=10, side=OrderSide.BUY)
+    ...     engine.add_order(price=1005, quantity=10, side=OrderSide.SELL)  # crosses
+    ...     bids, asks = engine.book(num_levels=5)
+
+Three things are worth knowing before you rely on it.
+
+**The engine owns a thread, so it has to be shut down.** Every operation publishes
+an event -- trades, adds, cancels, rejects -- to a background logger that writes to
+`log_file`. Use the engine as a context manager, or call :meth:`MatchingEngine.close`
+yourself. The log file is only complete once ``close()`` has returned.
+
+**The log is the only record of what happened.** Nothing is returned to Python
+beyond order ids and book views; in particular a rejected order comes back as
+``None`` and the reason for it appears solely in the log.
+
+**One engine is not thread-safe.** The book has no internal locking, and the
+methods deliberately hold the GIL, which is what keeps concurrent calls from
+corrupting it. An engine also cannot be used in a process forked from the one that
+created it -- the logger thread does not survive ``fork()`` -- and raises if tried.
+"""
+
+from ._core import MatchingEngine, OrderSide, OrderType, PriceLevel, RiskParams
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "MatchingEngine",
+    "OrderSide",
+    "OrderType",
+    "PriceLevel",
+    "RiskParams",
+    "__version__",
+]

python/matching_engine/_core.pyi

@@ -0,0 +1,79 @@
+"""Type stubs for the compiled extension module."""
+
+from types import TracebackType
+from typing import Optional
+
+class OrderSide:
+    BUY: OrderSide
+    SELL: OrderSide
+    @property
+    def name(self) -> str: ...
+    @property
+    def value(self) -> int: ...
+
+class OrderType:
+    LIMIT: OrderType
+    MARKET: OrderType
+    @property
+    def name(self) -> str: ...
+    @property
+    def value(self) -> int: ...
+
+class RiskParams:
+    max_allowed_quantity_quote: int
+    min_allowed_quantity_quote: int
+    max_price_book_top_deviation: int
+    def __init__(
+        self,
+        max_allowed_quantity_quote: int = ...,
+        min_allowed_quantity_quote: int = ...,
+        max_price_book_top_deviation: int = ...,
+    ) -> None: ...
+
+class PriceLevel:
+    @property
+    def price(self) -> int: ...
+    @property
+    def quantity(self) -> int: ...
+    @property
+    def order_ids(self) -> list[int]: ...
+
+class MatchingEngine:
+    def __init__(self, log_file: str, risk_params: RiskParams = ...) -> None: ...
+    def add_order(
+        self,
+        price: int,
+        quantity: int,
+        side: OrderSide,
+        order_type: OrderType = ...,
+    ) -> Optional[int]:
+        """The new order id, or None if pre-trade risk rejected it."""
+
+    def cancel_order(self, order_id: int) -> bool: ...
+    def modify_order(
+        self,
+        order_id: int,
+        quantity: int,
+        price: int,
+        side: OrderSide,
+        order_type: OrderType = ...,
+    ) -> Optional[int]:
+        """The surviving order id, or None if rejected or not in the book."""
+
+    def buy_levels(self, num_levels: int = 1) -> list[PriceLevel]: ...
+    def sell_levels(self, num_levels: int = 1) -> list[PriceLevel]: ...
+    def book(self, num_levels: int = 1) -> tuple[list[PriceLevel], list[PriceLevel]]: ...
+    def close(self) -> None: ...
+    @property
+    def closed(self) -> bool: ...
+    @property
+    def log_file(self) -> str: ...
+    @property
+    def risk_params(self) -> RiskParams: ...
+    def __enter__(self) -> MatchingEngine: ...
+    def __exit__(
+        self,
+        exc_type: Optional[type[BaseException]],
+        exc_value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> bool: ...

src/python/CMakeLists.txt

@@ -0,0 +1,40 @@
+# src/python/CMakeLists.txt
+# The pybind11 extension module. Configured only when ME_BUILD_PYTHON=ON.
+
+# pybind11 normally comes from the pip package that scikit-build-core places on
+# CMAKE_PREFIX_PATH. The FetchContent branch covers a bare developer configure
+# (cmake -S . -B build -DME_BUILD_PYTHON=ON) on a machine with no pip build
+# environment, where find_package would otherwise fail outright.
+find_package(pybind11 CONFIG QUIET)
+if(NOT pybind11_FOUND)
+    message(STATUS "pybind11 not found on CMAKE_PREFIX_PATH - fetching v2.13.6")
+    include(FetchContent)
+    FetchContent_Declare(
+        pybind11
+        GIT_REPOSITORY https://github.com/pybind/pybind11.git
+        GIT_TAG        v2.13.6
+    )
+    FetchContent_MakeAvailable(pybind11)
+endif()
+
+pybind11_add_module(_core MODULE module.cpp)
+target_compile_features(_core PRIVATE cxx_std_20)
+
+# project_warnings is omitted for the reason bench/CMakeLists.txt already records:
+# the strict set (-Wold-style-cast, -Wconversion, -Wsign-conversion) fires all over
+# the pybind11 headers, which cast through PyObject* and index with Py_ssize_t.
+#
+# project_options is omitted too. pybind11_add_module already applies hidden
+# visibility and LTO; scikit-build-core builds Release, which supplies -O3
+# -DNDEBUG; and -march=native on this one thin dispatch TU buys nothing, because
+# all the hot code lives in me_core, which IS built with project_options.
+target_link_libraries(_core PRIVATE
+    me_core
+    me_ds_lock          # LockQueue
+    me_event_handler    # Logger
+    Threads::Threads
+)
+
+# scikit-build-core assembles the wheel from the install tree, so the extension
+# has to land next to the pure-Python package.
+install(TARGETS _core DESTINATION matching_engine)