From 74f1cc1e8b8be04be2e04b4adeef6357ec9e06b6 Mon Sep 17 00:00:00 2001 From: rrohit2901 Date: Sun, 9 Aug 2026 11:03:44 +0530 Subject: [PATCH] Expose the matching engine to Python via pybind11 The engine could only be driven from C++. This adds an importable `matching_engine` module built with pybind11 and packaged with scikit-build-core, so `pip install .` yields a working extension. MatchingEngine cannot be bound directly. Its constructor takes an event queue that nothing drains unless a Logger is running on its own thread, so a caller handed the raw class could build an engine whose unbounded queue grows until it exhausts memory. Teardown is equally load-bearing: ~MatchingEngine publishes SESSION_CLOSE and Logger::stop() publishes the sentinel that ends the drain loop, so the engine must die before stop() or the last record never reaches the file. PyMatchingEngine owns the queue, logger, thread and engine, and makes that wiring non-optional. Only MatchingEngine is bound -- the class is templated on a template template parameter with a requires-clause, and that is the only instantiation used anywhere. Book views are returned as flat PriceLevel values rather than bound BookLevel objects. BookLevel is a live handle that re-reads the book on every accessor, so a Python object wrapping one would report the book as it is when read rather than when it was fetched. close() holds the GIL across engine_.reset() and releases it only around the thread join: another thread may be parked inside a bound method dereferencing engine_, and it can only stay parked while we hold the GIL. num_levels is validated because OrderBook's view takes an int but forwards to an unsigned parameter, so a negative count wrapped to about four billion and quietly returned the whole book. Build changes are option-guarded so every default reproduces the previous build byte for byte. Without the guards `pip install .` would clone googletest and Google Benchmark to build neither. me_core gains POSITION_INDEPENDENT_CODE because a static library cannot otherwise link into a shared module; measured A/B against a non-PIC build, the hot path P50s are unchanged. Co-Authored-By: Claude Opus 5 --- .github/workflows/CI.yml | 25 ++ .gitignore | 6 + CMakeLists.txt | 113 +++++--- pyproject.toml | 40 +++ python/matching_engine/__init__.py | 40 +++ python/matching_engine/_core.pyi | 79 ++++++ python/matching_engine/py.typed | 0 src/python/CMakeLists.txt | 40 +++ src/python/module.cpp | 405 +++++++++++++++++++++++++++++ tests/python/test_bindings.py | 282 ++++++++++++++++++++ 10 files changed, 998 insertions(+), 32 deletions(-) create mode 100644 pyproject.toml create mode 100644 python/matching_engine/__init__.py create mode 100644 python/matching_engine/_core.pyi create mode 100644 python/matching_engine/py.typed create mode 100644 src/python/CMakeLists.txt create mode 100644 src/python/module.cpp create mode 100644 tests/python/test_bindings.py diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index bd6da0d..6d09404 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -35,3 +35,28 @@ jobs: - name: Run tests run: ctest --test-dir build --output-on-failure + + python-bindings: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install build tools + run: | + sudo apt-get update + sudo apt-get install -y cmake ninja-build g++ + + # No FetchContent cache: with tests and benchmarks off, this configure + # clones nothing — pip supplies pybind11 to the isolated build environment. + - name: Build and install the extension + run: | + python -m pip install --upgrade pip + python -m pip install '.[test]' -v + + - name: Run Python tests + run: python -m pytest tests/python -v diff --git a/.gitignore b/.gitignore index 18fd62e..894e9f0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,9 @@ .claude build*/ data/ + +# Python +.venv/ +__pycache__/ +*.egg-info/ +.pytest_cache/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 615c44d..4eba401 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,7 +22,20 @@ endif() # Compile commands json for clangd / IDE integration set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - + +# ----------------------------------------------------------------------------- +# Build options. Every default reproduces the historical build exactly, so +# cmake -S . -B build && cmake --build build && ctest --test-dir build +# is unaffected by their existence. The wheel build (scikit-build-core, see +# pyproject.toml) turns tests/bench/apps off and python on, so `pip install .` +# never clones googletest or benchmark. +# ----------------------------------------------------------------------------- +option(ME_BUILD_TESTS "Build the GoogleTest unit tests" ON) +option(ME_BUILD_BENCH "Build the Google Benchmark suites" ON) +option(ME_BUILD_APPS "Build the me_main demo executable" ON) +option(ME_BUILD_PYTHON "Build the pybind11 extension module" OFF) +option(ME_NATIVE_ARCH "Tune Release builds for this CPU (-march=native)" ON) + # ----------------------------------------------------------------------------- # Warning flags applied via an interface target (modern pattern) # Link any target against `project_warnings` to inherit these. @@ -37,9 +50,13 @@ target_compile_options(project_warnings INTERFACE ) # Performance flags for Release (Debug already has -O0 -g by default) +# -march=native is behind ME_NATIVE_ARCH: a binary built with it dies with SIGILL +# on any machine older than the one that built it, which is fine locally and fatal +# for a redistributable wheel. With the option ON (the default) this expands +# exactly as it always did. add_library(project_options INTERFACE) target_compile_options(project_options INTERFACE - $<$:-O3 -march=native -DNDEBUG> + $<$:-O3 $<$:-march=native> -DNDEBUG> $<$:-O0 -g3 -fno-omit-frame-pointer> ) @@ -51,24 +68,32 @@ include(FetchContent) # The logger runs on its own thread and LockQueue uses std::mutex/condition_variable. find_package(Threads REQUIRED) +# Each dependency is fetched only when the thing that needs it is being built, so +# a Python-only configure (pip install .) clones neither. benchmark does not need +# googletest here because BENCHMARK_ENABLE_TESTING is forced OFF below. + # GoogleTest -FetchContent_Declare( - googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG v1.14.0 -) - +if(ME_BUILD_TESTS) + FetchContent_Declare( + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG v1.14.0 + ) + FetchContent_MakeAvailable(googletest) +endif() + # Google Benchmark -set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) -set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "" FORCE) -FetchContent_Declare( - benchmark - GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG v1.8.3 -) - -FetchContent_MakeAvailable(googletest benchmark) - +if(ME_BUILD_BENCH) + set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) + set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "" FORCE) + FetchContent_Declare( + benchmark + GIT_REPOSITORY https://github.com/google/benchmark.git + GIT_TAG v1.8.3 + ) + FetchContent_MakeAvailable(benchmark) +endif() + # ----------------------------------------------------------------------------- # Core library # ----------------------------------------------------------------------------- @@ -86,19 +111,29 @@ target_include_directories(me_core PUBLIC ) target_link_libraries(me_core PRIVATE project_warnings project_options) +# The Python extension is a shared object, so every object linked into it must be +# position independent; without this the module fails to link on x86-64 with +# R_X86_64_32S relocation errors. Set unconditionally rather than behind +# ME_BUILD_PYTHON so there is one build of me_core instead of a PIC and a non-PIC +# variant that can drift apart — the hot path has no globals, so the extra +# indirection does not show up in bench/. +set_target_properties(me_core PROPERTIES POSITION_INDEPENDENT_CODE ON) + # ----------------------------------------------------------------------------- # Main executable # ----------------------------------------------------------------------------- -add_executable(me_main src/main.cpp) -target_link_libraries(me_main PRIVATE - me_core - me_ds_lock # LockQueue - me_event_handler # Logger - Threads::Threads - project_warnings - project_options -) - +if(ME_BUILD_APPS) + add_executable(me_main src/main.cpp) + target_link_libraries(me_main PRIVATE + me_core + me_ds_lock # LockQueue + me_event_handler # Logger + Threads::Threads + project_warnings + project_options + ) +endif() + # ----------------------------------------------------------------------------- # Lock-based data structures (practice) — header-only. # Headers live in include/data_structures/; there is no .cpp, so this is an @@ -119,10 +154,24 @@ target_link_libraries(me_event_handler INTERFACE me_core Threads::Threads) # ----------------------------------------------------------------------------- # Tests # ----------------------------------------------------------------------------- -enable_testing() -add_subdirectory(tests) - +if(ME_BUILD_TESTS) + enable_testing() + add_subdirectory(tests) +endif() + # ----------------------------------------------------------------------------- # Benchmarks # ----------------------------------------------------------------------------- -add_subdirectory(bench) \ No newline at end of file +if(ME_BUILD_BENCH) + add_subdirectory(bench) +endif() + +# ----------------------------------------------------------------------------- +# Python bindings (opt-in; scikit-build-core sets ME_BUILD_PYTHON=ON) +# Last in the file on purpose: the module links me_ds_lock and me_event_handler, +# and unlike me_main a subdirectory cannot forward-reference targets that do not +# exist yet. +# ----------------------------------------------------------------------------- +if(ME_BUILD_PYTHON) + add_subdirectory(src/python) +endif() \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c67e978 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = ["scikit-build-core>=0.10", "pybind11>=2.12,<4"] +build-backend = "scikit_build_core.build" + +[project] +name = "matching-engine" +version = "0.1.0" # keep in step with project(VERSION) in CMakeLists.txt +description = "Python bindings for a single-symbol C++ matching engine" +requires-python = ">=3.9" +classifiers = [ + "Programming Language :: C++", + "Programming Language :: Python :: 3", + "Topic :: Office/Business :: Financial :: Investment", +] + +[project.optional-dependencies] +test = ["pytest>=7"] + +[tool.scikit-build] +minimum-version = "build-system.requires" +cmake.version = ">=3.20" +cmake.build-type = "Release" +# Under build*/, which .gitignore already covers. The wheel tag keeps it clear of +# a developer's own build/ tree and makes rebuilds incremental. +build-dir = "build/wheel-{wheel_tag}" +wheel.packages = ["python/matching_engine"] +sdist.exclude = ["logs/", ".vscode/", ".claude/"] + +# Only the extension is wanted here: the C++ tests and benchmarks would drag +# googletest and Google Benchmark in over the network for no reason. +# ME_NATIVE_ARCH is left at its default ON, which is right for `pip install .` on +# the machine that will run it. A redistributable wheel wants -DME_NATIVE_ARCH=OFF. +[tool.scikit-build.cmake.define] +ME_BUILD_PYTHON = "ON" +ME_BUILD_TESTS = "OFF" +ME_BUILD_BENCH = "OFF" +ME_BUILD_APPS = "OFF" + +[tool.pytest.ini_options] +testpaths = ["tests/python"] diff --git a/python/matching_engine/__init__.py b/python/matching_engine/__init__.py new file mode 100644 index 0000000..7cf062b --- /dev/null +++ b/python/matching_engine/__init__.py @@ -0,0 +1,40 @@ +"""A single-symbol matching engine, implemented in C++. + +Submit orders to a :class:`MatchingEngine` and it maintains a price-time priority +book, matching crossing orders on arrival and applying pre-trade risk checks: + + >>> from matching_engine import MatchingEngine, OrderSide + >>> with MatchingEngine("logs/session.log") as engine: + ... buy = engine.add_order(price=1005, quantity=10, side=OrderSide.BUY) + ... engine.add_order(price=1005, quantity=10, side=OrderSide.SELL) # crosses + ... bids, asks = engine.book(num_levels=5) + +Three things are worth knowing before you rely on it. + +**The engine owns a thread, so it has to be shut down.** Every operation publishes +an event -- trades, adds, cancels, rejects -- to a background logger that writes to +`log_file`. Use the engine as a context manager, or call :meth:`MatchingEngine.close` +yourself. The log file is only complete once ``close()`` has returned. + +**The log is the only record of what happened.** Nothing is returned to Python +beyond order ids and book views; in particular a rejected order comes back as +``None`` and the reason for it appears solely in the log. + +**One engine is not thread-safe.** The book has no internal locking, and the +methods deliberately hold the GIL, which is what keeps concurrent calls from +corrupting it. An engine also cannot be used in a process forked from the one that +created it -- the logger thread does not survive ``fork()`` -- and raises if tried. +""" + +from ._core import MatchingEngine, OrderSide, OrderType, PriceLevel, RiskParams + +__version__ = "0.1.0" + +__all__ = [ + "MatchingEngine", + "OrderSide", + "OrderType", + "PriceLevel", + "RiskParams", + "__version__", +] diff --git a/python/matching_engine/_core.pyi b/python/matching_engine/_core.pyi new file mode 100644 index 0000000..776363c --- /dev/null +++ b/python/matching_engine/_core.pyi @@ -0,0 +1,79 @@ +"""Type stubs for the compiled extension module.""" + +from types import TracebackType +from typing import Optional + +class OrderSide: + BUY: OrderSide + SELL: OrderSide + @property + def name(self) -> str: ... + @property + def value(self) -> int: ... + +class OrderType: + LIMIT: OrderType + MARKET: OrderType + @property + def name(self) -> str: ... + @property + def value(self) -> int: ... + +class RiskParams: + max_allowed_quantity_quote: int + min_allowed_quantity_quote: int + max_price_book_top_deviation: int + def __init__( + self, + max_allowed_quantity_quote: int = ..., + min_allowed_quantity_quote: int = ..., + max_price_book_top_deviation: int = ..., + ) -> None: ... + +class PriceLevel: + @property + def price(self) -> int: ... + @property + def quantity(self) -> int: ... + @property + def order_ids(self) -> list[int]: ... + +class MatchingEngine: + def __init__(self, log_file: str, risk_params: RiskParams = ...) -> None: ... + def add_order( + self, + price: int, + quantity: int, + side: OrderSide, + order_type: OrderType = ..., + ) -> Optional[int]: + """The new order id, or None if pre-trade risk rejected it.""" + + def cancel_order(self, order_id: int) -> bool: ... + def modify_order( + self, + order_id: int, + quantity: int, + price: int, + side: OrderSide, + order_type: OrderType = ..., + ) -> Optional[int]: + """The surviving order id, or None if rejected or not in the book.""" + + def buy_levels(self, num_levels: int = 1) -> list[PriceLevel]: ... + def sell_levels(self, num_levels: int = 1) -> list[PriceLevel]: ... + def book(self, num_levels: int = 1) -> tuple[list[PriceLevel], list[PriceLevel]]: ... + def close(self) -> None: ... + @property + def closed(self) -> bool: ... + @property + def log_file(self) -> str: ... + @property + def risk_params(self) -> RiskParams: ... + def __enter__(self) -> MatchingEngine: ... + def __exit__( + self, + exc_type: Optional[type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> bool: ... diff --git a/python/matching_engine/py.typed b/python/matching_engine/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/python/CMakeLists.txt b/src/python/CMakeLists.txt new file mode 100644 index 0000000..966b148 --- /dev/null +++ b/src/python/CMakeLists.txt @@ -0,0 +1,40 @@ +# src/python/CMakeLists.txt +# The pybind11 extension module. Configured only when ME_BUILD_PYTHON=ON. + +# pybind11 normally comes from the pip package that scikit-build-core places on +# CMAKE_PREFIX_PATH. The FetchContent branch covers a bare developer configure +# (cmake -S . -B build -DME_BUILD_PYTHON=ON) on a machine with no pip build +# environment, where find_package would otherwise fail outright. +find_package(pybind11 CONFIG QUIET) +if(NOT pybind11_FOUND) + message(STATUS "pybind11 not found on CMAKE_PREFIX_PATH - fetching v2.13.6") + include(FetchContent) + FetchContent_Declare( + pybind11 + GIT_REPOSITORY https://github.com/pybind/pybind11.git + GIT_TAG v2.13.6 + ) + FetchContent_MakeAvailable(pybind11) +endif() + +pybind11_add_module(_core MODULE module.cpp) +target_compile_features(_core PRIVATE cxx_std_20) + +# project_warnings is omitted for the reason bench/CMakeLists.txt already records: +# the strict set (-Wold-style-cast, -Wconversion, -Wsign-conversion) fires all over +# the pybind11 headers, which cast through PyObject* and index with Py_ssize_t. +# +# project_options is omitted too. pybind11_add_module already applies hidden +# visibility and LTO; scikit-build-core builds Release, which supplies -O3 +# -DNDEBUG; and -march=native on this one thin dispatch TU buys nothing, because +# all the hot code lives in me_core, which IS built with project_options. +target_link_libraries(_core PRIVATE + me_core + me_ds_lock # LockQueue + me_event_handler # Logger + Threads::Threads +) + +# scikit-build-core assembles the wheel from the install tree, so the extension +# has to land next to the pure-Python package. +install(TARGETS _core DESTINATION matching_engine) diff --git a/src/python/module.cpp b/src/python/module.cpp new file mode 100644 index 0000000..1e8d704 --- /dev/null +++ b/src/python/module.cpp @@ -0,0 +1,405 @@ +// Python bindings for the matching engine. +// +// The engine cannot be handed to Python as-is. MatchingEngine's constructor takes +// a shared_ptr to an event queue, and nothing drains that queue unless a Logger is +// running on its own thread — so a caller given the raw class could trivially build +// an engine whose unbounded queue grows until it exhausts memory. Teardown is +// equally load-bearing: ~MatchingEngine publishes SESSION_CLOSE, and Logger::stop() +// publishes the sentinel that ends the drain loop, so the engine must die BEFORE +// stop() or the last record never reaches the file. src/main.cpp gets this right by +// hand; PyMatchingEngine below is that same wiring made non-optional. +// +// Only MatchingEngine is bound: the class is templated on a template +// template parameter with a requires-clause, and pybind11 binds concrete +// instantiations only. That is also the only instantiation used anywhere in the +// repo. + +#include "MatchingEngine.h" +#include "Logger.h" +#include "lock_queue.h" + +#include +#include // vector/pair/optional/string conversions + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace py = pybind11; + +namespace { + +using EventQueue = LockQueue>; +using Engine = MatchingEngine; +using EventLogger = Logger; + +// --------------------------------------------------------------------------- +// Book views +// --------------------------------------------------------------------------- + +// BookLevel is a live handle onto the book, not a snapshot: it holds a +// shared_ptr and re-reads it on every accessor, so a Python object +// wrapping one would report whatever the book looks like when it is read rather +// than when it was fetched. Flat values captured at call time avoid that whole +// class of surprise. +struct PriceLevel { + int price; + int quantity; + std::vector order_ids; +}; + +std::vector snapshot(const std::vector>& levels) { + std::vector out; + out.reserve(levels.size()); + for (const auto& level : levels) { + if (!level) continue; + out.push_back(PriceLevel{level->getPrice(), level->getTotalQuantity(), level->getOrders()}); + } + return out; +} + +// --------------------------------------------------------------------------- +// The facade +// --------------------------------------------------------------------------- + +class PyMatchingEngine { + public: + PyMatchingEngine(std::string log_file, RiskParams risk_params) + : log_file_(std::move(log_file)), risk_(risk_params) { + // Logger opens the file on its own thread and, if the open fails, only + // warns on stderr and then drains every event into the void. From Python + // that would look like a silently empty log, so prove the path is usable + // here, while the failure can still be an exception. + prepareLogFile(log_file_); + + queue_ = std::make_shared(); + logger_ = std::make_unique(queue_, log_file_); + logger_thread_ = std::make_unique( + [logger = logger_.get()] { logger->readWriteLogs(); }); + + try { + engine_ = std::make_unique(queue_, risk_); // publishes SESSION_OPEN + } catch (...) { + // The logger thread is already blocked on the queue. Letting the + // members unwind now would destroy a joinable std::thread, which + // calls std::terminate — wind it down first. + logger_->stop(); + logger_thread_->join(); + throw; + } + + track(this); + } + + ~PyMatchingEngine() { + untrack(this); + // A throwing destructor during Python garbage collection terminates the + // process, so nothing may escape here. + try { + closeImpl(false); + } catch (...) { + } + // Last resort: destroying a joinable std::thread also terminates, so if + // closeImpl threw before the join, cut the thread loose instead. + if (logger_thread_ && logger_thread_->joinable()) logger_thread_->detach(); + } + + PyMatchingEngine(const PyMatchingEngine&) = delete; + PyMatchingEngine& operator=(const PyMatchingEngine&) = delete; + + // Called from Python, where the GIL is held on entry. + void close() { closeImpl(true); } + + bool closed() const { return closed_; } + const std::string& logFile() const { return log_file_; } + RiskParams riskParams() const { return risk_; } + + std::optional addOrder(int price, int quantity, OrderSide side, OrderType type) { + return engine().addOrder(price, quantity, type, side); + } + + bool cancelOrder(order_id_t order_id) { return engine().cancelOrder(order_id); } + + std::optional modifyOrder(order_id_t order_id, int quantity, int price, + OrderSide side, OrderType type) { + return engine().modifyOrder(order_id, quantity, price, side, type); + } + + std::vector buyLevels(int num_levels) { + return snapshot(engine().getBuySideView(checkLevels(num_levels))); + } + + std::vector sellLevels(int num_levels) { + return snapshot(engine().getSellSideView(checkLevels(num_levels))); + } + + std::pair, std::vector> book(int num_levels) { + const auto view = engine().getOrderBookView(checkLevels(num_levels)); + return {snapshot(view.first), snapshot(view.second)}; + } + + // Runs from the module's teardown hook, where the interpreter is finalizing; + // closeImpl is told not to touch the GIL there. + static void closeAll() { + std::lock_guard lock(registryMutex()); + for (auto* engine : registry()) { + try { + engine->closeImpl(false); + } catch (...) { + } + } + } + + // Registered once from the module initializer. + static void installForkHandlers() { + pthread_atfork(&PyMatchingEngine::beforeFork, + &PyMatchingEngine::afterForkParent, + &PyMatchingEngine::afterForkChild); + } + + private: + Engine& engine() { + if (forked_) + throw std::runtime_error( + "MatchingEngine cannot be used in a process forked from the one that " + "created it: the logger thread does not survive fork()"); + if (closed_ || !engine_) throw std::runtime_error("MatchingEngine is closed"); + return *engine_; + } + + // OrderBook::getBuySideView takes an int but forwards to + // OrderBookSide::getBookSideView(unsigned), so a negative count wraps to about + // four billion and quietly returns the entire book instead of failing. + static int checkLevels(int num_levels) { + if (num_levels < 1) throw std::invalid_argument("num_levels must be >= 1"); + return num_levels; + } + + // allow_gil_release is false when the interpreter is finalizing: releasing the + // GIL during teardown is not safe, and by then nothing else is running Python + // anyway. + void closeImpl(bool allow_gil_release) { + if (closed_) return; + closed_ = true; // set first, so a throw below cannot cause a second join + + if (forked_) { + abandonAfterFork(); + return; + } + + engine_.reset(); // ~MatchingEngine publishes SESSION_CLOSE + logger_->stop(); // the sentinel goes in behind it + + // The GIL is deliberately held across engine_.reset() above: another + // thread may be inside a bound method dereferencing engine_, and it can + // only be parked there while we hold the GIL. The join is the one + // blocking step, so it is the only thing worth releasing the GIL for. + if (allow_gil_release) { + py::gil_scoped_release unlocked; + logger_thread_->join(); + } else { + logger_thread_->join(); + } + + logger_thread_.reset(); + logger_.reset(); + } + + // A forked child inherits the queue and the engine but not the logger thread, + // and the queue's mutexes may have been held mid-push at the instant of the + // fork. Nothing here can be torn down: publishing SESSION_CLOSE could block on + // a mutex that will never be released, and joining a thread that does not + // exist in this process throws. Leak all of it, deliberately. + void abandonAfterFork() { + new std::shared_ptr(queue_); // pin the queue for good + (void) engine_.release(); + (void) logger_.release(); + (void) logger_thread_.release(); + } + + static void prepareLogFile(const std::string& path) { + const std::filesystem::path file{path}; + if (file.has_parent_path() && !file.parent_path().empty()) { + std::error_code ec; + std::filesystem::create_directories(file.parent_path(), ec); + if (ec) + throw std::runtime_error("cannot create log directory '" + + file.parent_path().string() + "': " + ec.message()); + } + std::ofstream probe{path, std::ios::out}; + if (!probe) throw std::runtime_error("cannot open log file '" + path + "' for writing"); + } + + // Function-local statics: the registry has to be alive before the first engine + // is constructed and after the last one is destroyed, and this ordering is + // guaranteed where a namespace-scope static's is not. + static std::mutex& registryMutex() { + static std::mutex mutex; + return mutex; + } + + static std::unordered_set& registry() { + static std::unordered_set engines; + return engines; + } + + static void track(PyMatchingEngine* engine) { + std::lock_guard lock(registryMutex()); + registry().insert(engine); + } + + static void untrack(PyMatchingEngine* engine) { + std::lock_guard lock(registryMutex()); + registry().erase(engine); + } + + // Standard pthread_atfork trio. The lock is taken before the fork and released + // on both sides, so the child never inherits a registry mutex that was held by + // a thread which no longer exists. + static void beforeFork() { registryMutex().lock(); } + static void afterForkParent() { registryMutex().unlock(); } + static void afterForkChild() { + for (auto* engine : registry()) engine->forked_ = true; + registryMutex().unlock(); + } + + std::string log_file_; + RiskParams risk_; + bool closed_ = false; + bool forked_ = false; + + std::shared_ptr queue_; + std::unique_ptr logger_; // Logger is immovable + // A unique_ptr, not a plain std::thread: in a forked child the thread does not + // exist, and both join() and detach() throw on it while destroying a joinable + // std::thread terminates. Only a pointer can be abandoned. + std::unique_ptr logger_thread_; + std::unique_ptr engine_; // MatchingEngine is immovable +}; + +std::string riskParamsRepr(const RiskParams& params) { + std::ostringstream out; + out << "RiskParams(max_allowed_quantity_quote=" << params.max_allowed_quantity_quote + << ", min_allowed_quantity_quote=" << params.min_allowed_quantity_quote + << ", max_price_book_top_deviation=" << params.max_price_book_top_deviation << ")"; + return out.str(); +} + +std::string priceLevelRepr(const PriceLevel& level) { + std::ostringstream out; + out << "PriceLevel(price=" << level.price << ", quantity=" << level.quantity + << ", orders=" << level.order_ids.size() << ")"; + return out.str(); +} + +} // namespace + +PYBIND11_MODULE(_core, m) { + m.doc() = "Single-symbol C++ matching engine."; + + PyMatchingEngine::installForkHandlers(); + + py::enum_(m, "OrderSide") + .value("BUY", OrderSide::BUY) + .value("SELL", OrderSide::SELL); + + py::enum_(m, "OrderType") + .value("LIMIT", OrderType::LIMIT) + .value("MARKET", OrderType::MARKET); + + const RiskParams defaults{}; + py::class_(m, "RiskParams", "Pre-trade risk limits, fixed at engine construction.") + // A lambda rather than py::init() so this does not depend on + // parenthesised aggregate initialisation. + .def(py::init([](int max_qty, int min_qty, int max_dev) { + return RiskParams{max_qty, min_qty, max_dev}; + }), + py::arg("max_allowed_quantity_quote") = defaults.max_allowed_quantity_quote, + py::arg("min_allowed_quantity_quote") = defaults.min_allowed_quantity_quote, + py::arg("max_price_book_top_deviation") = defaults.max_price_book_top_deviation) + .def_readwrite("max_allowed_quantity_quote", &RiskParams::max_allowed_quantity_quote) + .def_readwrite("min_allowed_quantity_quote", &RiskParams::min_allowed_quantity_quote) + .def_readwrite("max_price_book_top_deviation", &RiskParams::max_price_book_top_deviation) + .def("__repr__", &riskParamsRepr); + + py::class_(m, "PriceLevel", + "One price level, captured at the moment it was requested.") + .def_readonly("price", &PriceLevel::price) + .def_readonly("quantity", &PriceLevel::quantity) + .def_readonly("order_ids", &PriceLevel::order_ids) + .def("__repr__", &priceLevelRepr); + + py::class_(m, "MatchingEngine", R"doc( +A single-symbol matching engine writing its event log to `log_file`. + +Every operation publishes an event -- trades, adds, cancels, rejects -- to a +background logger thread that writes them to the log file. Nothing comes back to +Python: the log is the record. + +The engine owns that thread, so it must be shut down. Use it as a context manager +or call close(); the log file is only complete once close() has returned. + +Not thread-safe: the book has no internal locking, and these methods deliberately +hold the GIL, which is what serialises them. The engine also cannot be used in a +process forked from the one that created it -- the logger thread does not survive +fork() -- and will raise if you try. +)doc") + .def(py::init(), + py::arg("log_file"), py::arg("risk_params") = RiskParams{}) + .def("add_order", &PyMatchingEngine::addOrder, + py::arg("price"), py::arg("quantity"), py::arg("side"), + py::arg("order_type") = OrderType::LIMIT, + "Submit an order and match it against the book.\n\n" + "Returns the new order id, or None if pre-trade risk rejected it. The\n" + "reject reason is written to the log file and is not available here;\n" + "the limits that produced it are on the risk_params property.") + .def("cancel_order", &PyMatchingEngine::cancelOrder, py::arg("order_id"), + "Cancel a resting order. Returns False if it is not in the book.") + .def("modify_order", &PyMatchingEngine::modifyOrder, + py::arg("order_id"), py::arg("quantity"), py::arg("price"), py::arg("side"), + py::arg("order_type") = OrderType::LIMIT, + "Amend a resting order, then match it.\n\n" + "Returns the surviving order id -- a reprice retires the original and\n" + "books a replacement under a fresh id, so this is not always the id you\n" + "passed in. Returns None if risk rejected the amendment or the order is\n" + "not in the book; the two are indistinguishable here.") + .def("buy_levels", &PyMatchingEngine::buyLevels, py::arg("num_levels") = 1, + "The top num_levels bids, best first.") + .def("sell_levels", &PyMatchingEngine::sellLevels, py::arg("num_levels") = 1, + "The top num_levels asks, best first.") + .def("book", &PyMatchingEngine::book, py::arg("num_levels") = 1, + "Both sides at once as (bids, asks).") + // Deliberately no call_guard: close() needs the GIL for most of its work + // and releases it itself, only around the thread join. + .def("close", &PyMatchingEngine::close, + "Shut down the engine and flush the log. Idempotent.") + .def_property_readonly("closed", &PyMatchingEngine::closed) + .def_property_readonly("log_file", &PyMatchingEngine::logFile) + .def_property_readonly("risk_params", &PyMatchingEngine::riskParams) + .def("__enter__", [](PyMatchingEngine& self) -> PyMatchingEngine& { return self; }, + py::return_value_policy::reference_internal) + .def("__exit__", + [](PyMatchingEngine& self, const py::object&, const py::object&, const py::object&) { + self.close(); + return false; + }) + .def("__repr__", [](const PyMatchingEngine& self) { + return "" : ">"); + }); + + // Backstop for an engine still referenced when the interpreter exits: the + // capsule's destructor runs at module teardown. Without it the process would + // exit without joining the logger thread, losing the tail of the log. + m.add_object("_close_all_at_exit", py::capsule([] { PyMatchingEngine::closeAll(); })); +} diff --git a/tests/python/test_bindings.py b/tests/python/test_bindings.py new file mode 100644 index 0000000..bc3c418 --- /dev/null +++ b/tests/python/test_bindings.py @@ -0,0 +1,282 @@ +"""Tests for the pybind11 bindings. + +Two kinds of assertion here, because the engine reports almost nothing back to +Python. What a call returns -- an order id, None, a book view -- is checked +directly. Everything else the engine did is only visible in the log file, which is +parsed after the engine is closed. +""" + +import pytest + +from matching_engine import MatchingEngine, OrderSide, OrderType, RiskParams + + +@pytest.fixture +def log_path(tmp_path): + return tmp_path / "engine.log" + + +def read_log(path): + return [line for line in path.read_text().splitlines() if line] + + +# --------------------------------------------------------------------------- +# Orders +# --------------------------------------------------------------------------- + + +def test_add_order_returns_id_and_rests_in_book(log_path): + with MatchingEngine(str(log_path)) as engine: + order_id = engine.add_order(price=1005, quantity=10, side=OrderSide.BUY) + + assert isinstance(order_id, int) + levels = engine.buy_levels(num_levels=5) + assert len(levels) == 1 + assert levels[0].price == 1005 + assert levels[0].quantity == 10 + assert levels[0].order_ids == [order_id] + + +def test_crossing_order_trades_and_empties_the_book(log_path): + with MatchingEngine(str(log_path)) as engine: + engine.add_order(price=1005, quantity=10, side=OrderSide.BUY) + engine.add_order(price=1005, quantity=10, side=OrderSide.SELL) + + bids, asks = engine.book(num_levels=5) + assert bids == [] + assert asks == [] + + +def test_partial_fill_leaves_the_residual_resting(log_path): + with MatchingEngine(str(log_path)) as engine: + engine.add_order(price=1005, quantity=10, side=OrderSide.BUY) + engine.add_order(price=1005, quantity=4, side=OrderSide.SELL) + + bids, asks = engine.book(num_levels=5) + assert asks == [] + assert len(bids) == 1 + assert bids[0].quantity == 6 + + +def test_sell_side_is_reported_separately(log_path): + with MatchingEngine(str(log_path)) as engine: + engine.add_order(price=1010, quantity=7, side=OrderSide.SELL) + + assert engine.buy_levels() == [] + asks = engine.sell_levels(num_levels=5) + assert len(asks) == 1 + assert asks[0].price == 1010 + assert asks[0].quantity == 7 + + +def test_market_order_is_accepted(log_path): + with MatchingEngine(str(log_path)) as engine: + engine.add_order(price=1005, quantity=10, side=OrderSide.BUY) + order_id = engine.add_order( + price=1005, quantity=10, side=OrderSide.SELL, order_type=OrderType.MARKET + ) + + assert isinstance(order_id, int) + assert any("MARKET_ORDER_ADDED" in line for line in read_log(log_path)) + + +# --------------------------------------------------------------------------- +# Cancel and modify +# --------------------------------------------------------------------------- + + +def test_cancel_succeeds_once_then_fails(log_path): + with MatchingEngine(str(log_path)) as engine: + order_id = engine.add_order(price=1005, quantity=10, side=OrderSide.BUY) + + assert engine.cancel_order(order_id) is True + assert engine.cancel_order(order_id) is False + assert engine.buy_levels(num_levels=5) == [] + + +def test_modify_returns_the_surviving_id(log_path): + with MatchingEngine(str(log_path)) as engine: + order_id = engine.add_order(price=1005, quantity=10, side=OrderSide.BUY) + + # A requote can retire the original order and book a replacement, so the + # id that comes back is not necessarily the one passed in. + new_id = engine.modify_order( + order_id=order_id, quantity=15, price=1005, side=OrderSide.BUY + ) + + assert isinstance(new_id, int) + levels = engine.buy_levels(num_levels=5) + assert len(levels) == 1 + assert levels[0].quantity == 15 + + +def test_modify_of_unknown_order_returns_none(log_path): + with MatchingEngine(str(log_path)) as engine: + assert ( + engine.modify_order( + order_id=987654, quantity=10, price=1005, side=OrderSide.BUY + ) + is None + ) + + +# --------------------------------------------------------------------------- +# Risk +# --------------------------------------------------------------------------- + + +def test_zero_quantity_is_rejected_and_leaves_the_book_alone(log_path): + with MatchingEngine(str(log_path)) as engine: + assert engine.add_order(price=1005, quantity=0, side=OrderSide.BUY) is None + assert engine.buy_levels(num_levels=5) == [] + + +def test_price_far_from_top_of_book_is_rejected(log_path): + with MatchingEngine(str(log_path)) as engine: + engine.add_order(price=1005, quantity=10, side=OrderSide.BUY) + + assert engine.add_order(price=9999, quantity=10, side=OrderSide.BUY) is None + + +def test_custom_risk_params_are_applied_and_readable(log_path): + params = RiskParams(max_allowed_quantity_quote=50) + with MatchingEngine(str(log_path), params) as engine: + assert engine.risk_params.max_allowed_quantity_quote == 50 + assert engine.add_order(price=1005, quantity=51, side=OrderSide.BUY) is None + assert engine.add_order(price=1005, quantity=50, side=OrderSide.BUY) is not None + + +def test_risk_params_defaults(): + params = RiskParams() + assert params.max_allowed_quantity_quote == 100000 + assert params.min_allowed_quantity_quote == 1 + assert params.max_price_book_top_deviation == 1000 + + +# --------------------------------------------------------------------------- +# Argument validation +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("num_levels", [0, -1]) +def test_non_positive_num_levels_raises(log_path, num_levels): + # Regression test: the C++ view takes an int but forwards to an unsigned + # parameter, so a negative count used to wrap and return the whole book. + with MatchingEngine(str(log_path)) as engine: + with pytest.raises(ValueError): + engine.buy_levels(num_levels=num_levels) + with pytest.raises(ValueError): + engine.sell_levels(num_levels=num_levels) + with pytest.raises(ValueError): + engine.book(num_levels=num_levels) + + +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + + +def test_context_manager_closes(log_path): + with MatchingEngine(str(log_path)) as engine: + assert engine.closed is False + assert engine.closed is True + + +def test_close_is_idempotent(log_path): + engine = MatchingEngine(str(log_path)) + engine.close() + engine.close() + assert engine.closed is True + + +def test_use_after_close_raises(log_path): + engine = MatchingEngine(str(log_path)) + engine.close() + + with pytest.raises(RuntimeError): + engine.add_order(price=1005, quantity=10, side=OrderSide.BUY) + with pytest.raises(RuntimeError): + engine.cancel_order(1) + with pytest.raises(RuntimeError): + engine.buy_levels() + + +def test_log_file_property(log_path): + with MatchingEngine(str(log_path)) as engine: + assert engine.log_file == str(log_path) + + +def test_missing_log_directory_is_created(tmp_path): + nested = tmp_path / "a" / "b" / "engine.log" + with MatchingEngine(str(nested)) as engine: + engine.add_order(price=1005, quantity=10, side=OrderSide.BUY) + + assert nested.exists() + + +def test_unwritable_log_path_raises_at_construction(tmp_path): + # A directory can never be opened for writing, so this fails at the probe in + # the constructor rather than silently dropping every event on the logger + # thread. + with pytest.raises(RuntimeError): + MatchingEngine(str(tmp_path)) + + +# --------------------------------------------------------------------------- +# The event log +# --------------------------------------------------------------------------- + + +def test_session_markers_bracket_the_log(log_path): + with MatchingEngine(str(log_path)) as engine: + engine.add_order(price=1005, quantity=10, side=OrderSide.BUY) + + lines = read_log(log_path) + # SESSION_CLOSE landing last is the whole teardown contract in one assertion: + # the engine has to be destroyed before the logger is told to stop, or the + # sentinel overtakes this record and it never reaches the file. + assert "SESSION_OPEN" in lines[0] + assert "SESSION_CLOSE" in lines[-1] + + +def test_trade_is_logged_with_both_order_ids(log_path): + with MatchingEngine(str(log_path)) as engine: + buy_id = engine.add_order(price=1005, quantity=10, side=OrderSide.BUY) + sell_id = engine.add_order(price=1005, quantity=10, side=OrderSide.SELL) + + trades = [line for line in read_log(log_path) if "TRADE_EVENT" in line] + assert len(trades) == 1 + assert f"BUY_ID: {buy_id}" in trades[0] + assert f"SELL_ID: {sell_id}" in trades[0] + assert "TRADE_QTY: 10" in trades[0] + assert "TRADE_PRICE: 1005" in trades[0] + + +def test_reject_reason_is_recoverable_from_the_log(log_path): + # add_order only says None. This is where the reason actually lives. + with MatchingEngine(str(log_path)) as engine: + engine.add_order(price=1005, quantity=0, side=OrderSide.BUY) + + rejects = [line for line in read_log(log_path) if "ORDER_REJECTED" in line] + assert len(rejects) == 1 + assert "REASON: QUANTITY_BELOW_MIN" in rejects[0] + + +def test_cancel_is_logged(log_path): + with MatchingEngine(str(log_path)) as engine: + order_id = engine.add_order(price=1005, quantity=10, side=OrderSide.BUY) + engine.cancel_order(order_id) + + cancels = [line for line in read_log(log_path) if "ORDER_CANCELLED" in line] + assert len(cancels) == 1 + assert f"ORDER_ID: {order_id}" in cancels[0] + + +def test_log_is_complete_only_after_close(log_path): + engine = MatchingEngine(str(log_path)) + engine.add_order(price=1005, quantity=10, side=OrderSide.BUY) + engine.close() + + lines = read_log(log_path) + assert any("LIMIT_ORDER_ADDED" in line for line in lines) + assert "SESSION_CLOSE" in lines[-1]