From 9a117a3086e2f77e3ab8f6b6cbd44c5e5f341db9 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Mon, 8 Jun 2026 08:28:37 -0400 Subject: [PATCH] chore(profiling): native C++/Rust py3.15 ABI support --- .../profiling/cmake/FindLibNative.cmake | 8 +- .../profiling/cmake/NativeHeaders.cmake | 28 +++ .../profiling/dd_wrapper/src/sample.cpp | 10 +- .../datadog/profiling/ddup/CMakeLists.txt | 13 +- .../datadog/profiling/stack/CMakeLists.txt | 10 +- .../stack/echion/echion/cpython/tasks.h | 53 ++++- .../profiling/stack/fuzz/CMakeLists.txt | 6 +- .../profiling/stack/src/echion/frame.cc | 39 ++-- .../profiling/stack/test/CMakeLists.txt | 33 ++- .../test/test_cpython_layout_contracts.cpp | 201 ++++++++++++++++++ .../stack/test/test_frame_state_315.cpp | 146 +++++++++++++ 11 files changed, 505 insertions(+), 42 deletions(-) create mode 100644 ddtrace/internal/datadog/profiling/cmake/NativeHeaders.cmake create mode 100644 ddtrace/internal/datadog/profiling/stack/test/test_cpython_layout_contracts.cpp create mode 100644 ddtrace/internal/datadog/profiling/stack/test/test_frame_state_315.cpp diff --git a/ddtrace/internal/datadog/profiling/cmake/FindLibNative.cmake b/ddtrace/internal/datadog/profiling/cmake/FindLibNative.cmake index edb6f9bd833..a8d3ded915a 100644 --- a/ddtrace/internal/datadog/profiling/cmake/FindLibNative.cmake +++ b/ddtrace/internal/datadog/profiling/cmake/FindLibNative.cmake @@ -21,10 +21,10 @@ endif() message(WARNING "SOURCE_LIB_DIR: ${SOURCE_LIB_DIR}") message(WARNING "LIBRARY_NAME: ${LIBRARY_NAME}") -# We expect the native extension to be built and installed the headers in the following directory. It is configured in -# setup.py by setting CARGO_TARGET_DIR environment variable. -set(SOURCE_INCLUDE_DIR - ${CMAKE_SOURCE_DIR}/../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include) +# Resolves NATIVE_HEADERS_DIR (libdatadog's generated C headers for the active Python minor). See NativeHeaders.cmake +# for the resolution policy. +include(NativeHeaders) +set(SOURCE_INCLUDE_DIR "${NATIVE_HEADERS_DIR}") set(DEST_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}) set(DEST_INCLUDE_DIR ${DEST_LIB_DIR}/include) diff --git a/ddtrace/internal/datadog/profiling/cmake/NativeHeaders.cmake b/ddtrace/internal/datadog/profiling/cmake/NativeHeaders.cmake new file mode 100644 index 00000000000..cf28ae21dd5 --- /dev/null +++ b/ddtrace/internal/datadog/profiling/cmake/NativeHeaders.cmake @@ -0,0 +1,28 @@ +# Resolves NATIVE_HEADERS_DIR — the absolute path to libdatadog's generated C headers (produced by the Rust crate under +# src/native/ and written to target./include). +# +# Primary source: setup.py passes -DRUST_GENERATED_HEADERS_DIR= to every CMake invocation via +# _get_common_cmake_args. Whenever that variable is set, we trust it. +# +# Fallback: build_standalone.sh does NOT pass RUST_GENERATED_HEADERS_DIR, so we compute a path relative to this module's +# own location. Callers must have already invoked find_package(Python3) so that Python3_VERSION_MAJOR/_MINOR are +# defined; the fallback uses those to pick the right per-minor target directory (matching setup.py's CARGO_TARGET_DIR +# layout). +# +# Consumers must have "${CMAKE_CURRENT_SOURCE_DIR}/../cmake" on CMAKE_MODULE_PATH before calling include(NativeHeaders). + +if(DEFINED RUST_GENERATED_HEADERS_DIR) + set(NATIVE_HEADERS_DIR "${RUST_GENERATED_HEADERS_DIR}") +else() + if(NOT DEFINED Python3_VERSION_MAJOR OR NOT DEFINED Python3_VERSION_MINOR) + message( + FATAL_ERROR + "NativeHeaders: RUST_GENERATED_HEADERS_DIR is not set and Python3_VERSION_MAJOR/_MINOR are undefined. " + "Call find_package(Python3) before include(NativeHeaders), or pass -DRUST_GENERATED_HEADERS_DIR " + "(as setup.py does).") + endif() + get_filename_component( + NATIVE_HEADERS_DIR + "${CMAKE_CURRENT_LIST_DIR}/../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include" + ABSOLUTE) +endif() diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp index 02b0797252d..0e57f1f85c9 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp @@ -1,10 +1,14 @@ -#include "sample.hpp" - +// TODO(py-315): Python.h must be included first, before any system or project headers. +// CPython's pyconfig.h defines _POSIX_C_SOURCE and _XOPEN_SOURCE to their current +// POSIX standard values (202405L on 3.15+). If system headers (included transitively +// via libdatadog_helpers.hpp → features.h) are pulled in first, they define older +// values (200809L), and pyconfig.h's later redefinition triggers -Werror on GCC/Clang. #define PY_SSIZE_T_CLEAN - #include #include +#include "sample.hpp" + #include "libdatadog_helpers.hpp" #include "profiler_state.hpp" #include "pymacro.hpp" diff --git a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt index 3ac62b0f1f5..5f1cbce606b 100644 --- a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt @@ -60,7 +60,9 @@ add_library(${EXTENSION_NAME} SHARED ${DDUP_CPP_SRC}) add_ddup_config(${EXTENSION_NAME}) # Cython generates code that produces errors for the following, so relax compile options -target_compile_options(${EXTENSION_NAME} PRIVATE -Wno-old-style-cast -Wno-shadow -Wno-address) +# -Wno-missing-field-initializers: Python 3.15 added tp_iteritem to PyTypeObject; Cython doesn't initialize it yet +target_compile_options(${EXTENSION_NAME} PRIVATE -Wno-old-style-cast -Wno-shadow -Wno-address + -Wno-missing-field-initializers) # cmake may mutate the name of the library (e.g., lib- and -.so for dynamic libraries). This suppresses that behavior, # which is required to ensure all paths can be inferred correctly by setup.py. @@ -87,11 +89,10 @@ elseif(UNIX) endif() endif() -target_include_directories( - ${EXTENSION_NAME} - PRIVATE ../dd_wrapper/include - ../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include/ - ${Datadog_INCLUDE_DIRS} ${Python3_INCLUDE_DIRS}) +include(NativeHeaders) + +target_include_directories(${EXTENSION_NAME} PRIVATE ../dd_wrapper/include "${NATIVE_HEADERS_DIR}" + ${Datadog_INCLUDE_DIRS} ${Python3_INCLUDE_DIRS}) target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper) diff --git a/ddtrace/internal/datadog/profiling/stack/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack/CMakeLists.txt index 21011f4aa43..2f8f927b2e1 100644 --- a/ddtrace/internal/datadog/profiling/stack/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack/CMakeLists.txt @@ -94,14 +94,16 @@ add_clangtidy_target(${EXTENSION_NAME}) # Never build with native unwinding, since this is not currently used target_compile_definitions(${EXTENSION_NAME} PRIVATE UNWIND_NATIVE_DISABLE) +# Resolves NATIVE_HEADERS_DIR (libdatadog's generated C headers for the active Python minor). See +# cmake/NativeHeaders.cmake for the resolution policy. +include(NativeHeaders) + # Includes; echion and python are marked "system" to suppress warnings target_include_directories( ${EXTENSION_NAME} PRIVATE .. # include dd_wrapper from the root in order to make its paths transparent in the code include) -target_include_directories( - ${EXTENSION_NAME} SYSTEM - PRIVATE ${Python3_INCLUDE_DIRS} echion include/vendored include/util - ../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include/) +target_include_directories(${EXTENSION_NAME} SYSTEM PRIVATE ${Python3_INCLUDE_DIRS} echion include/vendored + include/util "${NATIVE_HEADERS_DIR}") # Echion sources need to be given the current platform if(APPLE) diff --git a/ddtrace/internal/datadog/profiling/stack/echion/echion/cpython/tasks.h b/ddtrace/internal/datadog/profiling/stack/echion/echion/cpython/tasks.h index b91e44106f9..eebb777a109 100644 --- a/ddtrace/internal/datadog/profiling/stack/echion/echion/cpython/tasks.h +++ b/ddtrace/internal/datadog/profiling/stack/echion/echion/cpython/tasks.h @@ -224,8 +224,57 @@ extern "C" #define RESUME_QUICK INSTRUMENTED_RESUME #endif -#if PY_VERSION_HEX >= 0x030e0000 - // Python 3.14+: Use stackpointer and _PyStackRef +#if PY_VERSION_HEX >= 0x030f0000 + // Python 3.15+: FRAME_SUSPENDED_YIELD_FROM_LOCKED is a new frame state for + // generators that are locked during a yield-from in free-threaded builds. + // In GIL builds this state is unreachable, so we only check it under + // Py_GIL_DISABLED. All other logic is identical to 3.14 (stackpointer/_PyStackRef). + + inline PyObject* PyGen_yf(PyGenObject* gen, PyObject* frame_addr) + { + if (gen->gi_frame_state != FRAME_SUSPENDED_YIELD_FROM +#ifdef Py_GIL_DISABLED + && gen->gi_frame_state != FRAME_SUSPENDED_YIELD_FROM_LOCKED +#endif + ) { + return nullptr; + } + + _PyInterpreterFrame frame; + if (copy_type(frame_addr, frame)) { + return nullptr; + } + + PyCodeObject code; + auto code_addr = reinterpret_cast(BITS_TO_PTR_MASKED(frame.f_executable)); + if (copy_type(code_addr, code)) { + return nullptr; + } + + uintptr_t frame_addr_uint = reinterpret_cast(frame_addr); + uintptr_t localsplus_addr = frame_addr_uint + offsetof(_PyInterpreterFrame, localsplus); + uintptr_t stackbase_addr = localsplus_addr + code.co_nlocalsplus * sizeof(_PyStackRef); + + uintptr_t stackpointer_addr = reinterpret_cast(frame.stackpointer); + if (stackpointer_addr <= stackbase_addr) { + return nullptr; + } + + int stacktop = static_cast((stackpointer_addr - stackbase_addr) / sizeof(_PyStackRef)); + if (stacktop < 1 || stacktop > MAX_STACK_SIZE) { + return nullptr; + } + + _PyStackRef top_ref; + if (copy_type(reinterpret_cast(stackpointer_addr - sizeof(_PyStackRef)), top_ref)) { + return nullptr; + } + + return BITS_TO_PTR_MASKED(top_ref); + } + +#elif PY_VERSION_HEX >= 0x030e0000 + // Python 3.14: Use stackpointer and _PyStackRef inline PyObject* PyGen_yf(PyGenObject* gen, PyObject* frame_addr) { diff --git a/ddtrace/internal/datadog/profiling/stack/fuzz/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack/fuzz/CMakeLists.txt index 6a6e1d76ece..58b5cda87cc 100644 --- a/ddtrace/internal/datadog/profiling/stack/fuzz/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack/fuzz/CMakeLists.txt @@ -28,10 +28,8 @@ function(add_fuzz_target TARGET_NAME) # Include paths: ../.. is the profiling root (for "dd_wrapper/include/..." paths), ../include is for stack headers. target_include_directories(${TARGET_NAME} PRIVATE ../.. ../include) - target_include_directories( - ${TARGET_NAME} SYSTEM - PRIVATE ${Python3_INCLUDE_DIRS} ../echion ../include/vendored ../include/util - ../../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include/) + target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${Python3_INCLUDE_DIRS} ../echion ../include/vendored + ../include/util "${NATIVE_HEADERS_DIR}") # Ensure echion headers take the fuzz hook in vm.h target_compile_definitions(${TARGET_NAME} PRIVATE ECHION_FUZZING) diff --git a/ddtrace/internal/datadog/profiling/stack/src/echion/frame.cc b/ddtrace/internal/datadog/profiling/stack/src/echion/frame.cc index 2d608e08c77..9e38e1a4aa1 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/echion/frame.cc +++ b/ddtrace/internal/datadog/profiling/stack/src/echion/frame.cc @@ -111,22 +111,31 @@ Frame::read(EchionSampler& echion, PyObject* frame_addr, PyObject** prev_addr) frame_addr = &iframe; #if PY_VERSION_HEX >= 0x030c0000 + // _PyInterpreterFrame.owner is stored as char, not the _frameowner enum + // itself, so -Wswitch can't enforce exhaustiveness. test_cpython_layout + // _contracts static_asserts the enum values we rely on; an unknown owner + // here means CPython grew a new value and is treated as an error. + switch (frame_addr->owner) { + case FRAME_OWNED_BY_THREAD: + case FRAME_OWNED_BY_GENERATOR: + break; // valid live Python frame — proceed with frame reading + case FRAME_OWNED_BY_FRAME_OBJECT: + return ErrorKind::FrameError; // frame belongs to a PyFrameObject, not executing +#if PY_VERSION_HEX < 0x030f0000 + case FRAME_OWNED_BY_CSTACK: // C shim frame (removed in 3.15) +#endif #if PY_VERSION_HEX >= 0x030e0000 - // Python 3.14 introduced FRAME_OWNED_BY_INTERPRETER, and frames of this - // type are also ignored by the upstream profiler. - // See - // https://github.com/python/cpython/blob/ebf955df7a89ed0c7968f79faec1de49f61ed7cb/Modules/_remote_debugging_module.c#L2134 - if (frame_addr->owner == FRAME_OWNED_BY_CSTACK || frame_addr->owner == FRAME_OWNED_BY_INTERPRETER) { -#else - if (frame_addr->owner == FRAME_OWNED_BY_CSTACK) { -#endif // PY_VERSION_HEX >= 0x030e0000 - *prev_addr = frame_addr->previous; - // This is a C frame, we just need to ignore it - return std::ref(C_FRAME); - } - - if (frame_addr->owner != FRAME_OWNED_BY_THREAD && frame_addr->owner != FRAME_OWNED_BY_GENERATOR) { - return ErrorKind::FrameError; + case FRAME_OWNED_BY_INTERPRETER: +#endif + // C/interpreter-managed frame — skip it and follow the frame chain. + // FRAME_OWNED_BY_INTERPRETER introduced in 3.14; FRAME_OWNED_BY_CSTACK + // present in 3.12–3.14, removed in 3.15. + // See + // https://github.com/python/cpython/blob/ebf955df7a89ed0c7968f79faec1de49f61ed7cb/Modules/_remote_debugging_module.c#L2134 + *prev_addr = frame_addr->previous; + return std::ref(C_FRAME); + default: + return ErrorKind::FrameError; } #endif // PY_VERSION_HEX >= 0x030c0000 diff --git a/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt index face972eb7b..ee34a40224f 100644 --- a/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt @@ -36,13 +36,16 @@ if(DO_VALGRIND) endif() function(dd_wrapper_add_test name) - # Optional keyword argument: INSTALL_SUBDIR Installs the test binary to test// instead of test/. Use for - # version-specific binaries (e.g. INSTALL_SUBDIR py315) to prevent CI artifact collisions when build_base_venvs runs - # in parallel across Python versions and GitLab merges all artifacts into a shared directory. cmake_parse_arguments(_ARG "" "INSTALL_SUBDIR" "" ${ARGN}) set(_SOURCES ${_ARG_UNPARSED_ARGUMENTS}) add_executable(${name} ${_SOURCES}) - target_include_directories(${name} PRIVATE ../include) + # Replicate the include dirs that ${EXTENSION_NAME} sets PRIVATE (so they don't propagate to test targets): stack's + # own headers, the profiling root (for "dd_wrapper/include/sample.hpp"), Python, echion, and the libdatadog + # Rust-generated headers (NATIVE_HEADERS_DIR, set by the parent CMakeLists). + target_include_directories(${name} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../include" + "${CMAKE_CURRENT_SOURCE_DIR}/../..") + target_include_directories(${name} SYSTEM PRIVATE ${Python3_INCLUDE_DIRS} "${CMAKE_CURRENT_SOURCE_DIR}/../echion" + "${NATIVE_HEADERS_DIR}") # this has to refer to the stack extension name to properly link against target_link_libraries(${name} PRIVATE gmock gtest_main ${EXTENSION_NAME}) @@ -71,6 +74,14 @@ function(dd_wrapper_add_test name) gtest_discover_tests(${name} DISCOVERY_MODE PRE_TEST) # Delay test discovery until test execution to avoid running # sanitizer-built executables during build + # Echion headers (vm.h, tasks.h) require PL_DARWIN or PL_LINUX to define proc_ref_t and copy_type. These are set + # PRIVATE on ${EXTENSION_NAME} and don't propagate to test targets, so we replicate the logic here. + if(APPLE) + target_compile_definitions(${name} PRIVATE PL_DARWIN) + else() + target_compile_definitions(${name} PRIVATE PL_LINUX) + endif() + # This is supplemental artifact so make sure to install it in the right place if(INPLACE_LIB_INSTALL_DIR) set(LIB_INSTALL_DIR "${INPLACE_LIB_INSTALL_DIR}") @@ -92,3 +103,17 @@ dd_wrapper_add_test(test_origin_task_links test_origin_task_links.cpp) dd_wrapper_add_test(test_alt_stack_ownership test_alt_stack_ownership.cpp) # ThreadAltStack lives in the vendored echion header tree. target_include_directories(test_alt_stack_ownership PRIVATE ../echion) +# test_frame_state_315 validates the PyFrameState renumbering and related frame-internals changes introduced in Python +# 3.15. All test bodies are gated on PY_VERSION_HEX >= 0x030f0000, so building on older versions would produce an empty +# test binary. +if(Python3_VERSION VERSION_GREATER_EQUAL "3.15") + # INSTALL_SUBDIR py315 keeps this binary out of the shared test/ directory. build_base_venvs runs in parallel for + # all Python versions; GitLab merges their artifacts into one directory for downstream jobs. Without isolation the + # py3.15-compiled binary (RPATH -> libpython3.15) would crash when the pytest gtest plugin tried to run it in a + # py3.10 environment. + dd_wrapper_add_test(test_frame_state_315 test_frame_state_315.cpp INSTALL_SUBDIR py315) + # Route copy_memory through echion_fuzz_copy_memory so tests can assert whether the state guard allows execution to + # reach the copy site. + target_compile_definitions(test_frame_state_315 PRIVATE ECHION_FUZZING) +endif() +dd_wrapper_add_test(test_cpython_layout_contracts test_cpython_layout_contracts.cpp) diff --git a/ddtrace/internal/datadog/profiling/stack/test/test_cpython_layout_contracts.cpp b/ddtrace/internal/datadog/profiling/stack/test/test_cpython_layout_contracts.cpp new file mode 100644 index 00000000000..625b46b1e2b --- /dev/null +++ b/ddtrace/internal/datadog/profiling/stack/test/test_cpython_layout_contracts.cpp @@ -0,0 +1,201 @@ +// Compile-time contracts for CPython internal enum values that echion depends on. +// +// Each static_assert fires at *compile time* against the actual CPython headers — +// if CPython renumbers or removes an enum value the build breaks immediately, +// before any test runner is invoked. The matching gtest TEST() wrappers surface +// the same checks as human-readable failures in CI output. +// +// Update these blocks when adding support for a new CPython minor version: +// 1. Add a new versioned block with the new values. +// 2. Adjust the upper-bound on the previous block if values changed. +// 3. Run the build against the new CPython to confirm all assertions pass. +// +// Enums covered: +// _frameowner (pycore_interpframe_structs.h, 3.12+) +// PyFrameState (pycore_frame.h, 3.11+) + +#define PY_SSIZE_T_CLEAN +#define Py_BUILD_CORE +#include + +#include + +#if PY_VERSION_HEX >= 0x030e0000 +// Python 3.14+: frame internals split into separate headers; +// _frameowner is in pycore_interpframe_structs.h (new in 3.14). +#include +#include +#include +#elif PY_VERSION_HEX >= 0x030b0000 +// Python 3.11-3.13: _frameowner enum lives directly in pycore_frame.h. +// pycore_interpframe_structs.h does not exist on these versions. +#include +#endif + +// echion/vm.h defines proc_ref_t, which the stub below requires. +#include + +// Stub: echion's remote-memory callback is referenced at link time via vm.h. +// Always returns failure — no live process attached in unit tests. +extern "C" int +echion_fuzz_copy_memory(proc_ref_t /*proc_ref*/, const void* /*addr*/, ssize_t /*len*/, void* /*buf*/) +{ + return -1; +} + +// ───────────────────────────────────────────────────────────────────────────── +// _frameowner enum (pycore_interpframe_structs.h, introduced in 3.12) +// ───────────────────────────────────────────────────────────────────────────── + +// 3.12 – 3.13: four members, CSTACK=3, no INTERPRETER +#if PY_VERSION_HEX >= 0x030c0000 && PY_VERSION_HEX < 0x030e0000 +static_assert(FRAME_OWNED_BY_THREAD == 0, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_THREAD changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_GENERATOR == 1, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_GENERATOR changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_FRAME_OBJECT == 2, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_FRAME_OBJECT changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_CSTACK == 3, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_CSTACK changed value — update frame.cc owner switch"); + +TEST(FrameOwnerEnum_312_313, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_OWNED_BY_THREAD, 0); + EXPECT_EQ(FRAME_OWNED_BY_GENERATOR, 1); + EXPECT_EQ(FRAME_OWNED_BY_FRAME_OBJECT, 2); + EXPECT_EQ(FRAME_OWNED_BY_CSTACK, 3); +} +#endif // 3.12 – 3.13 + +// 3.14: five members, INTERPRETER added (=3), CSTACK bumped to 4 +#if PY_VERSION_HEX >= 0x030e0000 && PY_VERSION_HEX < 0x030f0000 +static_assert(FRAME_OWNED_BY_THREAD == 0, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_THREAD changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_GENERATOR == 1, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_GENERATOR changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_FRAME_OBJECT == 2, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_FRAME_OBJECT changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_INTERPRETER == 3, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_INTERPRETER changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_CSTACK == 4, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_CSTACK changed value — update frame.cc owner switch"); + +TEST(FrameOwnerEnum_314, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_OWNED_BY_THREAD, 0); + EXPECT_EQ(FRAME_OWNED_BY_GENERATOR, 1); + EXPECT_EQ(FRAME_OWNED_BY_FRAME_OBJECT, 2); + EXPECT_EQ(FRAME_OWNED_BY_INTERPRETER, 3); + EXPECT_EQ(FRAME_OWNED_BY_CSTACK, 4); +} +#endif // 3.14 + +// 3.15+: FRAME_OWNED_BY_CSTACK removed +#if PY_VERSION_HEX >= 0x030f0000 +static_assert(FRAME_OWNED_BY_THREAD == 0, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_THREAD changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_GENERATOR == 1, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_GENERATOR changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_FRAME_OBJECT == 2, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_FRAME_OBJECT changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_INTERPRETER == 3, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_INTERPRETER changed value — update frame.cc owner switch"); +// FRAME_OWNED_BY_CSTACK intentionally not listed — it was removed in 3.15. +// If this file compiles without error, CPython has not re-introduced it. + +TEST(FrameOwnerEnum_315, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_OWNED_BY_THREAD, 0); + EXPECT_EQ(FRAME_OWNED_BY_GENERATOR, 1); + EXPECT_EQ(FRAME_OWNED_BY_FRAME_OBJECT, 2); + EXPECT_EQ(FRAME_OWNED_BY_INTERPRETER, 3); +} +#endif // 3.15+ + +// ───────────────────────────────────────────────────────────────────────────── +// PyFrameState / gi_frame_state (pycore_frame.h, introduced in 3.11) +// ───────────────────────────────────────────────────────────────────────────── + +// 3.11 – 3.12: negative-valued range, no FRAME_SUSPENDED_YIELD_FROM yet. +// FRAME_SUSPENDED_YIELD_FROM was introduced in 3.13 (CPython gh-104210), which +// also shifted FRAME_CREATED and FRAME_SUSPENDED one slot more negative. +#if PY_VERSION_HEX >= 0x030b0000 && PY_VERSION_HEX < 0x030d0000 +static_assert(FRAME_CREATED == -2, + "TODO(py-315): PyFrameState::FRAME_CREATED changed value — update tasks.h PyGen_yf and tasks.cc"); +static_assert(FRAME_SUSPENDED == -1, "TODO(py-315): PyFrameState::FRAME_SUSPENDED changed value"); +static_assert(FRAME_EXECUTING == 0, + "TODO(py-315): PyFrameState::FRAME_EXECUTING changed value — update tasks.cc gen_is_running check"); +// FRAME_COMPLETED == 1 is not used by echion directly; omitted intentionally. +static_assert(FRAME_CLEARED == 4, + "TODO(py-315): PyFrameState::FRAME_CLEARED changed value — update tasks.cc gi_frame_state check"); + +TEST(PyFrameStateEnum_311_312, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_CREATED, -2); + EXPECT_EQ(FRAME_SUSPENDED, -1); + EXPECT_EQ(FRAME_EXECUTING, 0); + EXPECT_EQ(FRAME_CLEARED, 4); +} +#endif // 3.11 – 3.12 + +// 3.13 – 3.14: negative-valued range, FRAME_SUSPENDED_YIELD_FROM added (-1), +// pushing FRAME_CREATED to -3 and FRAME_SUSPENDED to -2. +#if PY_VERSION_HEX >= 0x030d0000 && PY_VERSION_HEX < 0x030f0000 +static_assert(FRAME_CREATED == -3, + "TODO(py-315): PyFrameState::FRAME_CREATED changed value — update tasks.h PyGen_yf and tasks.cc"); +static_assert(FRAME_SUSPENDED == -2, "TODO(py-315): PyFrameState::FRAME_SUSPENDED changed value"); +static_assert(FRAME_SUSPENDED_YIELD_FROM == -1, + "TODO(py-315): PyFrameState::FRAME_SUSPENDED_YIELD_FROM changed value — update tasks.h PyGen_yf"); +static_assert(FRAME_EXECUTING == 0, + "TODO(py-315): PyFrameState::FRAME_EXECUTING changed value — update tasks.cc gen_is_running check"); +// FRAME_COMPLETED == 1 is not used by echion directly; omitted intentionally. +static_assert(FRAME_CLEARED == 4, + "TODO(py-315): PyFrameState::FRAME_CLEARED changed value — update tasks.cc gi_frame_state check"); + +TEST(PyFrameStateEnum_313_314, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_CREATED, -3); + EXPECT_EQ(FRAME_SUSPENDED, -2); + EXPECT_EQ(FRAME_SUSPENDED_YIELD_FROM, -1); + EXPECT_EQ(FRAME_EXECUTING, 0); + EXPECT_EQ(FRAME_CLEARED, 4); +} +#endif // 3.13 – 3.14 + +// 3.15+: all values renumbered, COMPLETED removed +#if PY_VERSION_HEX >= 0x030f0000 +static_assert(FRAME_CREATED == 0, + "TODO(py-315): PyFrameState::FRAME_CREATED changed value — update tasks.h PyGen_yf and tasks.cc"); +static_assert(FRAME_SUSPENDED == 1, "TODO(py-315): PyFrameState::FRAME_SUSPENDED changed value"); +static_assert(FRAME_SUSPENDED_YIELD_FROM == 2, + "TODO(py-315): PyFrameState::FRAME_SUSPENDED_YIELD_FROM changed value — update tasks.h PyGen_yf"); +// value 3 is FRAME_SUSPENDED_YIELD_FROM_LOCKED in free-threaded builds (see below) +static_assert(FRAME_EXECUTING == 4, + "TODO(py-315): PyFrameState::FRAME_EXECUTING changed value — update tasks.cc gen_is_running check"); +static_assert(FRAME_CLEARED == 5, + "TODO(py-315): PyFrameState::FRAME_CLEARED changed value — update tasks.cc gi_frame_state check"); +// FRAME_COMPLETED intentionally not listed — it was removed in 3.15. + +#ifdef Py_GIL_DISABLED +static_assert( + FRAME_SUSPENDED_YIELD_FROM_LOCKED == 3, + "TODO(py-315): FRAME_SUSPENDED_YIELD_FROM_LOCKED changed value — update tasks.h PyGen_yf (Py_GIL_DISABLED)"); +#endif + +TEST(PyFrameStateEnum_315, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_CREATED, 0); + EXPECT_EQ(FRAME_SUSPENDED, 1); + EXPECT_EQ(FRAME_SUSPENDED_YIELD_FROM, 2); + EXPECT_EQ(FRAME_EXECUTING, 4); + EXPECT_EQ(FRAME_CLEARED, 5); +} + +#ifdef Py_GIL_DISABLED +TEST(PyFrameStateEnum_315_NoGIL, LockedYieldFromValueMatchesExpected) +{ + EXPECT_EQ(FRAME_SUSPENDED_YIELD_FROM_LOCKED, 3); +} +#endif + +#endif // 3.15+ diff --git a/ddtrace/internal/datadog/profiling/stack/test/test_frame_state_315.cpp b/ddtrace/internal/datadog/profiling/stack/test/test_frame_state_315.cpp new file mode 100644 index 00000000000..bb356d5fc8a --- /dev/null +++ b/ddtrace/internal/datadog/profiling/stack/test/test_frame_state_315.cpp @@ -0,0 +1,146 @@ +// Unit tests for Python 3.15 frame-state guard changes. +// +// Covered: +// 1. Static assertions on renumbered PyFrameState enum values (3.15+). +// 2. PyGen_yf returns nullptr for FRAME_SUSPENDED_YIELD_FROM_LOCKED in GIL builds (3.15+). +// 3. PyGen_yf enters the body for FRAME_SUSPENDED_YIELD_FROM even after the 3.15 guard change. +// 4. PyGen_yf returns nullptr for all non-suspended states (3.15+). +// +// Memory stub: copy_type/copy_generic call echion_fuzz_copy_memory. We define it here to +// always return failure (-1), which is the correct outcome when no real Python process is +// attached. All code paths that reach a copy_type call will return nullptr safely. + +#define PY_SSIZE_T_CLEAN +#define Py_BUILD_CORE +#include + +#include + +#include +#include + +#if PY_VERSION_HEX >= 0x030e0000 +#include +#include +#include +#include +#include +#endif + +#include +#include +#include + +// Counter tracking how many times copy_memory was invoked. Reset before each +// PyGen_yf call so tests can assert whether the state guard allowed execution +// to reach the copy site (>0) or filtered it out first (0). +// Must be declared before echion headers use ECHION_FUZZING to route copy_memory +// through this stub; atomic so future parallel-test runs stay race-free. +static std::atomic g_copy_attempts{ 0 }; + +extern "C" int +echion_fuzz_copy_memory(proc_ref_t /*proc_ref*/, const void* /*addr*/, ssize_t /*len*/, void* /*buf*/) +{ + g_copy_attempts.fetch_add(1, std::memory_order_relaxed); + return -1; // always fail — no live process attached +} + +// ───────────────────────────────────────────────────────────────────────────── +// 1. Compile-time enum value assertions (3.15+ only) +// ───────────────────────────────────────────────────────────────────────────── + +#if PY_VERSION_HEX >= 0x030f0000 + +// PyFrameState was renumbered in 3.15. Verify our understanding matches reality so +// that any future CPython change is caught immediately at compile time. +static_assert(FRAME_CREATED == 0, "FRAME_CREATED should be 0 in Python 3.15"); +static_assert(FRAME_SUSPENDED == 1, "FRAME_SUSPENDED should be 1 in Python 3.15"); +static_assert(FRAME_SUSPENDED_YIELD_FROM == 2, "FRAME_SUSPENDED_YIELD_FROM should be 2 in Python 3.15"); +static_assert(FRAME_EXECUTING == 4, "FRAME_EXECUTING should be 4 in Python 3.15"); +static_assert(FRAME_CLEARED == 5, "FRAME_CLEARED should be 5 in Python 3.15"); + +#ifdef Py_GIL_DISABLED +// FRAME_SUSPENDED_YIELD_FROM_LOCKED only exists when building against a free-threaded Python. +static_assert(FRAME_SUSPENDED_YIELD_FROM_LOCKED == 3, "FRAME_SUSPENDED_YIELD_FROM_LOCKED should be 3 in Python 3.15"); +#endif // Py_GIL_DISABLED + +TEST(PyFrameState315, EnumValuesMatchExpected) +{ + // Runtime counterpart of the static_asserts above — provides a readable failure + // message in the test output if run against an unexpected Python build. + EXPECT_EQ(FRAME_CREATED, 0); + EXPECT_EQ(FRAME_SUSPENDED, 1); + EXPECT_EQ(FRAME_SUSPENDED_YIELD_FROM, 2); + EXPECT_EQ(FRAME_EXECUTING, 4); + EXPECT_EQ(FRAME_CLEARED, 5); +} + +// ───────────────────────────────────────────────────────────────────────────── +// 2. PyGen_yf state-check tests (3.15+) +// +// PyGenObject::gi_frame_state is an int (signed). We set only that field; all +// other fields are zero-initialised. We pass nullptr as frame_addr so that if the +// state check passes, copy_type will immediately fail and return nullptr — which +// means any test that expects nullptr is still correct regardless of whether the +// state check or the copy fails first. +// ───────────────────────────────────────────────────────────────────────────── + +static PyGenObject +make_fake_gen(int frame_state) +{ + PyGenObject gen{}; + gen.gi_frame_state = frame_state; + return gen; +} + +#ifndef Py_GIL_DISABLED + +TEST(PyGenYf315GilBuild, LockedStateIgnored) +{ + // FRAME_SUSPENDED_YIELD_FROM_LOCKED (value 3) must NOT be treated as a + // suspended-yield-from state in GIL builds. PyGen_yf should return nullptr + // immediately from the state guard without attempting any memory read. + g_copy_attempts.store(0, std::memory_order_relaxed); + auto gen = make_fake_gen(3 /* FRAME_SUSPENDED_YIELD_FROM_LOCKED value */); + PyObject* result = PyGen_yf(&gen, nullptr); + EXPECT_EQ(result, nullptr); + EXPECT_EQ(g_copy_attempts.load(std::memory_order_relaxed), 0) + << "state guard must filter FRAME_SUSPENDED_YIELD_FROM_LOCKED without any copy attempt"; +} + +TEST(PyGenYf315GilBuild, SuspendedYieldFromEntersBody) +{ + // FRAME_SUSPENDED_YIELD_FROM must still be recognised as a suspended state. + // The state guard passes, execution enters the body, and copy_type(nullptr, frame) + // immediately fails — confirming the guard did NOT filter out this state. + g_copy_attempts.store(0, std::memory_order_relaxed); + auto gen = make_fake_gen(FRAME_SUSPENDED_YIELD_FROM); + PyObject* result = PyGen_yf(&gen, nullptr); + EXPECT_EQ(result, nullptr); // copy_type fails on nullptr frame_addr + EXPECT_GT(g_copy_attempts.load(std::memory_order_relaxed), 0) + << "FRAME_SUSPENDED_YIELD_FROM must pass the state guard and attempt a copy"; +} + +#endif // !Py_GIL_DISABLED + +// Parametrised: non-suspended states must all return nullptr immediately. +class PyGenYf315OtherStates : public ::testing::TestWithParam +{}; + +TEST_P(PyGenYf315OtherStates, ReturnsNull) +{ + g_copy_attempts.store(0, std::memory_order_relaxed); + auto gen = make_fake_gen(GetParam()); + EXPECT_EQ(PyGen_yf(&gen, nullptr), nullptr); + EXPECT_EQ(g_copy_attempts.load(std::memory_order_relaxed), 0) + << "non-suspended states must be filtered by the state guard without any copy attempt"; +} + +INSTANTIATE_TEST_SUITE_P(NonSuspendedStates, + PyGenYf315OtherStates, + ::testing::Values(FRAME_CREATED, // 0 + FRAME_EXECUTING, // 4 + FRAME_CLEARED // 5 + )); + +#endif // PY_VERSION_HEX >= 0x030f0000