From d9bcea5b857d623fde1633f27b68f5e775526c3f Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Fri, 5 Jun 2026 13:26:20 -0400 Subject: [PATCH 01/14] feat(profiling): add INSTALL_SUBDIR keyword to dd_wrapper_add_test Allows version-specific native test binaries to install into a subdir of the shared test/ directory (e.g. INSTALL_SUBDIR py315 -> test/py315/). build_base_venvs runs in parallel across all Python versions and GitLab merges all artifacts into a single directory for downstream jobs. Without isolation a binary compiled for pyX.Y (RPATH -> libpythonX.Y) lands in the shared test/ directory and crashes when the pytest gtest plugin tries to run it against a different Python runtime. Callers that do not pass INSTALL_SUBDIR are unaffected. Co-Authored-By: Claude Sonnet 4.6 --- .../datadog/profiling/stack/test/CMakeLists.txt | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt index 173db9145f2..790cd418e39 100644 --- a/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt @@ -36,7 +36,12 @@ if(DO_VALGRIND) endif() function(dd_wrapper_add_test name) - add_executable(${name} ${ARGN}) + # Optional keyword argument: INSTALL_SUBDIR Installs the test binary to test// instead of test/. Use for + # version-specific binaries (e.g. INSTALL_SUBDIR py315) to prevent CI artifact collisions when build_base_venvs runs + # in parallel across Python versions and GitLab merges all artifacts into a shared directory. + cmake_parse_arguments(_ARG "" "INSTALL_SUBDIR" "" ${ARGN}) + set(_SOURCES ${_ARG_UNPARSED_ARGUMENTS}) + add_executable(${name} ${_SOURCES}) target_include_directories(${name} PRIVATE ../include) # this has to refer to the stack extension name to properly link against target_link_libraries(${name} PRIVATE gmock gtest_main ${EXTENSION_NAME}) @@ -72,7 +77,11 @@ function(dd_wrapper_add_test name) endif() if(LIB_INSTALL_DIR) - install(TARGETS ${name} RUNTIME DESTINATION ${LIB_INSTALL_DIR}/../test) + if(_ARG_INSTALL_SUBDIR) + install(TARGETS ${name} RUNTIME DESTINATION ${LIB_INSTALL_DIR}/../test/${_ARG_INSTALL_SUBDIR}) + else() + install(TARGETS ${name} RUNTIME DESTINATION ${LIB_INSTALL_DIR}/../test) + endif() endif() endfunction() From e6e1f72babc2e11cdf30db666e57a7863ecc3748 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Mon, 8 Jun 2026 08:28:37 -0400 Subject: [PATCH 02/14] chore(profiling): native C++/Rust py3.15 ABI support --- .../profiling/cmake/FindLibNative.cmake | 8 +- .../profiling/cmake/NativeHeaders.cmake | 28 +++ .../profiling/dd_wrapper/src/sample.cpp | 10 +- .../datadog/profiling/ddup/CMakeLists.txt | 13 +- .../datadog/profiling/stack/CMakeLists.txt | 10 +- .../stack/echion/echion/cpython/tasks.h | 53 ++++- .../profiling/stack/fuzz/CMakeLists.txt | 6 +- .../profiling/stack/src/echion/frame.cc | 39 ++-- .../profiling/stack/test/CMakeLists.txt | 33 ++- .../test/test_cpython_layout_contracts.cpp | 201 ++++++++++++++++++ .../stack/test/test_frame_state_315.cpp | 146 +++++++++++++ 11 files changed, 505 insertions(+), 42 deletions(-) create mode 100644 ddtrace/internal/datadog/profiling/cmake/NativeHeaders.cmake create mode 100644 ddtrace/internal/datadog/profiling/stack/test/test_cpython_layout_contracts.cpp create mode 100644 ddtrace/internal/datadog/profiling/stack/test/test_frame_state_315.cpp diff --git a/ddtrace/internal/datadog/profiling/cmake/FindLibNative.cmake b/ddtrace/internal/datadog/profiling/cmake/FindLibNative.cmake index edb6f9bd833..a8d3ded915a 100644 --- a/ddtrace/internal/datadog/profiling/cmake/FindLibNative.cmake +++ b/ddtrace/internal/datadog/profiling/cmake/FindLibNative.cmake @@ -21,10 +21,10 @@ endif() message(WARNING "SOURCE_LIB_DIR: ${SOURCE_LIB_DIR}") message(WARNING "LIBRARY_NAME: ${LIBRARY_NAME}") -# We expect the native extension to be built and installed the headers in the following directory. It is configured in -# setup.py by setting CARGO_TARGET_DIR environment variable. -set(SOURCE_INCLUDE_DIR - ${CMAKE_SOURCE_DIR}/../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include) +# Resolves NATIVE_HEADERS_DIR (libdatadog's generated C headers for the active Python minor). See NativeHeaders.cmake +# for the resolution policy. +include(NativeHeaders) +set(SOURCE_INCLUDE_DIR "${NATIVE_HEADERS_DIR}") set(DEST_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}) set(DEST_INCLUDE_DIR ${DEST_LIB_DIR}/include) diff --git a/ddtrace/internal/datadog/profiling/cmake/NativeHeaders.cmake b/ddtrace/internal/datadog/profiling/cmake/NativeHeaders.cmake new file mode 100644 index 00000000000..cf28ae21dd5 --- /dev/null +++ b/ddtrace/internal/datadog/profiling/cmake/NativeHeaders.cmake @@ -0,0 +1,28 @@ +# Resolves NATIVE_HEADERS_DIR — the absolute path to libdatadog's generated C headers (produced by the Rust crate under +# src/native/ and written to target./include). +# +# Primary source: setup.py passes -DRUST_GENERATED_HEADERS_DIR= to every CMake invocation via +# _get_common_cmake_args. Whenever that variable is set, we trust it. +# +# Fallback: build_standalone.sh does NOT pass RUST_GENERATED_HEADERS_DIR, so we compute a path relative to this module's +# own location. Callers must have already invoked find_package(Python3) so that Python3_VERSION_MAJOR/_MINOR are +# defined; the fallback uses those to pick the right per-minor target directory (matching setup.py's CARGO_TARGET_DIR +# layout). +# +# Consumers must have "${CMAKE_CURRENT_SOURCE_DIR}/../cmake" on CMAKE_MODULE_PATH before calling include(NativeHeaders). + +if(DEFINED RUST_GENERATED_HEADERS_DIR) + set(NATIVE_HEADERS_DIR "${RUST_GENERATED_HEADERS_DIR}") +else() + if(NOT DEFINED Python3_VERSION_MAJOR OR NOT DEFINED Python3_VERSION_MINOR) + message( + FATAL_ERROR + "NativeHeaders: RUST_GENERATED_HEADERS_DIR is not set and Python3_VERSION_MAJOR/_MINOR are undefined. " + "Call find_package(Python3) before include(NativeHeaders), or pass -DRUST_GENERATED_HEADERS_DIR " + "(as setup.py does).") + endif() + get_filename_component( + NATIVE_HEADERS_DIR + "${CMAKE_CURRENT_LIST_DIR}/../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include" + ABSOLUTE) +endif() diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp index 17fb356be6b..c47eda66625 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp @@ -1,10 +1,14 @@ -#include "sample.hpp" - +// TODO(py-315): Python.h must be included first, before any system or project headers. +// CPython's pyconfig.h defines _POSIX_C_SOURCE and _XOPEN_SOURCE to their current +// POSIX standard values (202405L on 3.15+). If system headers (included transitively +// via libdatadog_helpers.hpp → features.h) are pulled in first, they define older +// values (200809L), and pyconfig.h's later redefinition triggers -Werror on GCC/Clang. #define PY_SSIZE_T_CLEAN - #include #include +#include "sample.hpp" + #include "libdatadog_helpers.hpp" #include "profiler_state.hpp" #include "pymacro.hpp" diff --git a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt index 3ac62b0f1f5..5f1cbce606b 100644 --- a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt @@ -60,7 +60,9 @@ add_library(${EXTENSION_NAME} SHARED ${DDUP_CPP_SRC}) add_ddup_config(${EXTENSION_NAME}) # Cython generates code that produces errors for the following, so relax compile options -target_compile_options(${EXTENSION_NAME} PRIVATE -Wno-old-style-cast -Wno-shadow -Wno-address) +# -Wno-missing-field-initializers: Python 3.15 added tp_iteritem to PyTypeObject; Cython doesn't initialize it yet +target_compile_options(${EXTENSION_NAME} PRIVATE -Wno-old-style-cast -Wno-shadow -Wno-address + -Wno-missing-field-initializers) # cmake may mutate the name of the library (e.g., lib- and -.so for dynamic libraries). This suppresses that behavior, # which is required to ensure all paths can be inferred correctly by setup.py. @@ -87,11 +89,10 @@ elseif(UNIX) endif() endif() -target_include_directories( - ${EXTENSION_NAME} - PRIVATE ../dd_wrapper/include - ../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include/ - ${Datadog_INCLUDE_DIRS} ${Python3_INCLUDE_DIRS}) +include(NativeHeaders) + +target_include_directories(${EXTENSION_NAME} PRIVATE ../dd_wrapper/include "${NATIVE_HEADERS_DIR}" + ${Datadog_INCLUDE_DIRS} ${Python3_INCLUDE_DIRS}) target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper) diff --git a/ddtrace/internal/datadog/profiling/stack/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack/CMakeLists.txt index 21011f4aa43..2f8f927b2e1 100644 --- a/ddtrace/internal/datadog/profiling/stack/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack/CMakeLists.txt @@ -94,14 +94,16 @@ add_clangtidy_target(${EXTENSION_NAME}) # Never build with native unwinding, since this is not currently used target_compile_definitions(${EXTENSION_NAME} PRIVATE UNWIND_NATIVE_DISABLE) +# Resolves NATIVE_HEADERS_DIR (libdatadog's generated C headers for the active Python minor). See +# cmake/NativeHeaders.cmake for the resolution policy. +include(NativeHeaders) + # Includes; echion and python are marked "system" to suppress warnings target_include_directories( ${EXTENSION_NAME} PRIVATE .. # include dd_wrapper from the root in order to make its paths transparent in the code include) -target_include_directories( - ${EXTENSION_NAME} SYSTEM - PRIVATE ${Python3_INCLUDE_DIRS} echion include/vendored include/util - ../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include/) +target_include_directories(${EXTENSION_NAME} SYSTEM PRIVATE ${Python3_INCLUDE_DIRS} echion include/vendored + include/util "${NATIVE_HEADERS_DIR}") # Echion sources need to be given the current platform if(APPLE) diff --git a/ddtrace/internal/datadog/profiling/stack/echion/echion/cpython/tasks.h b/ddtrace/internal/datadog/profiling/stack/echion/echion/cpython/tasks.h index b91e44106f9..eebb777a109 100644 --- a/ddtrace/internal/datadog/profiling/stack/echion/echion/cpython/tasks.h +++ b/ddtrace/internal/datadog/profiling/stack/echion/echion/cpython/tasks.h @@ -224,8 +224,57 @@ extern "C" #define RESUME_QUICK INSTRUMENTED_RESUME #endif -#if PY_VERSION_HEX >= 0x030e0000 - // Python 3.14+: Use stackpointer and _PyStackRef +#if PY_VERSION_HEX >= 0x030f0000 + // Python 3.15+: FRAME_SUSPENDED_YIELD_FROM_LOCKED is a new frame state for + // generators that are locked during a yield-from in free-threaded builds. + // In GIL builds this state is unreachable, so we only check it under + // Py_GIL_DISABLED. All other logic is identical to 3.14 (stackpointer/_PyStackRef). + + inline PyObject* PyGen_yf(PyGenObject* gen, PyObject* frame_addr) + { + if (gen->gi_frame_state != FRAME_SUSPENDED_YIELD_FROM +#ifdef Py_GIL_DISABLED + && gen->gi_frame_state != FRAME_SUSPENDED_YIELD_FROM_LOCKED +#endif + ) { + return nullptr; + } + + _PyInterpreterFrame frame; + if (copy_type(frame_addr, frame)) { + return nullptr; + } + + PyCodeObject code; + auto code_addr = reinterpret_cast(BITS_TO_PTR_MASKED(frame.f_executable)); + if (copy_type(code_addr, code)) { + return nullptr; + } + + uintptr_t frame_addr_uint = reinterpret_cast(frame_addr); + uintptr_t localsplus_addr = frame_addr_uint + offsetof(_PyInterpreterFrame, localsplus); + uintptr_t stackbase_addr = localsplus_addr + code.co_nlocalsplus * sizeof(_PyStackRef); + + uintptr_t stackpointer_addr = reinterpret_cast(frame.stackpointer); + if (stackpointer_addr <= stackbase_addr) { + return nullptr; + } + + int stacktop = static_cast((stackpointer_addr - stackbase_addr) / sizeof(_PyStackRef)); + if (stacktop < 1 || stacktop > MAX_STACK_SIZE) { + return nullptr; + } + + _PyStackRef top_ref; + if (copy_type(reinterpret_cast(stackpointer_addr - sizeof(_PyStackRef)), top_ref)) { + return nullptr; + } + + return BITS_TO_PTR_MASKED(top_ref); + } + +#elif PY_VERSION_HEX >= 0x030e0000 + // Python 3.14: Use stackpointer and _PyStackRef inline PyObject* PyGen_yf(PyGenObject* gen, PyObject* frame_addr) { diff --git a/ddtrace/internal/datadog/profiling/stack/fuzz/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack/fuzz/CMakeLists.txt index 37846a592f3..c1c4d9f8061 100644 --- a/ddtrace/internal/datadog/profiling/stack/fuzz/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack/fuzz/CMakeLists.txt @@ -29,10 +29,8 @@ function(add_fuzz_target TARGET_NAME) # Include paths: ../.. is the profiling root (for "dd_wrapper/include/..." paths), ../include is for stack headers. target_include_directories(${TARGET_NAME} PRIVATE ../.. ../include) - target_include_directories( - ${TARGET_NAME} SYSTEM - PRIVATE ${Python3_INCLUDE_DIRS} ../echion ../include/vendored ../include/util - ../../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include/) + target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${Python3_INCLUDE_DIRS} ../echion ../include/vendored + ../include/util "${NATIVE_HEADERS_DIR}") # Ensure echion headers take the fuzz hook in vm.h target_compile_definitions(${TARGET_NAME} PRIVATE ECHION_FUZZING) diff --git a/ddtrace/internal/datadog/profiling/stack/src/echion/frame.cc b/ddtrace/internal/datadog/profiling/stack/src/echion/frame.cc index 2d608e08c77..9e38e1a4aa1 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/echion/frame.cc +++ b/ddtrace/internal/datadog/profiling/stack/src/echion/frame.cc @@ -111,22 +111,31 @@ Frame::read(EchionSampler& echion, PyObject* frame_addr, PyObject** prev_addr) frame_addr = &iframe; #if PY_VERSION_HEX >= 0x030c0000 + // _PyInterpreterFrame.owner is stored as char, not the _frameowner enum + // itself, so -Wswitch can't enforce exhaustiveness. test_cpython_layout + // _contracts static_asserts the enum values we rely on; an unknown owner + // here means CPython grew a new value and is treated as an error. + switch (frame_addr->owner) { + case FRAME_OWNED_BY_THREAD: + case FRAME_OWNED_BY_GENERATOR: + break; // valid live Python frame — proceed with frame reading + case FRAME_OWNED_BY_FRAME_OBJECT: + return ErrorKind::FrameError; // frame belongs to a PyFrameObject, not executing +#if PY_VERSION_HEX < 0x030f0000 + case FRAME_OWNED_BY_CSTACK: // C shim frame (removed in 3.15) +#endif #if PY_VERSION_HEX >= 0x030e0000 - // Python 3.14 introduced FRAME_OWNED_BY_INTERPRETER, and frames of this - // type are also ignored by the upstream profiler. - // See - // https://github.com/python/cpython/blob/ebf955df7a89ed0c7968f79faec1de49f61ed7cb/Modules/_remote_debugging_module.c#L2134 - if (frame_addr->owner == FRAME_OWNED_BY_CSTACK || frame_addr->owner == FRAME_OWNED_BY_INTERPRETER) { -#else - if (frame_addr->owner == FRAME_OWNED_BY_CSTACK) { -#endif // PY_VERSION_HEX >= 0x030e0000 - *prev_addr = frame_addr->previous; - // This is a C frame, we just need to ignore it - return std::ref(C_FRAME); - } - - if (frame_addr->owner != FRAME_OWNED_BY_THREAD && frame_addr->owner != FRAME_OWNED_BY_GENERATOR) { - return ErrorKind::FrameError; + case FRAME_OWNED_BY_INTERPRETER: +#endif + // C/interpreter-managed frame — skip it and follow the frame chain. + // FRAME_OWNED_BY_INTERPRETER introduced in 3.14; FRAME_OWNED_BY_CSTACK + // present in 3.12–3.14, removed in 3.15. + // See + // https://github.com/python/cpython/blob/ebf955df7a89ed0c7968f79faec1de49f61ed7cb/Modules/_remote_debugging_module.c#L2134 + *prev_addr = frame_addr->previous; + return std::ref(C_FRAME); + default: + return ErrorKind::FrameError; } #endif // PY_VERSION_HEX >= 0x030c0000 diff --git a/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt index 790cd418e39..ea366f84c44 100644 --- a/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack/test/CMakeLists.txt @@ -36,13 +36,16 @@ if(DO_VALGRIND) endif() function(dd_wrapper_add_test name) - # Optional keyword argument: INSTALL_SUBDIR Installs the test binary to test// instead of test/. Use for - # version-specific binaries (e.g. INSTALL_SUBDIR py315) to prevent CI artifact collisions when build_base_venvs runs - # in parallel across Python versions and GitLab merges all artifacts into a shared directory. cmake_parse_arguments(_ARG "" "INSTALL_SUBDIR" "" ${ARGN}) set(_SOURCES ${_ARG_UNPARSED_ARGUMENTS}) add_executable(${name} ${_SOURCES}) - target_include_directories(${name} PRIVATE ../include) + # Replicate the include dirs that ${EXTENSION_NAME} sets PRIVATE (so they don't propagate to test targets): stack's + # own headers, the profiling root (for "dd_wrapper/include/sample.hpp"), Python, echion, and the libdatadog + # Rust-generated headers (NATIVE_HEADERS_DIR, set by the parent CMakeLists). + target_include_directories(${name} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../include" + "${CMAKE_CURRENT_SOURCE_DIR}/../..") + target_include_directories(${name} SYSTEM PRIVATE ${Python3_INCLUDE_DIRS} "${CMAKE_CURRENT_SOURCE_DIR}/../echion" + "${NATIVE_HEADERS_DIR}") # this has to refer to the stack extension name to properly link against target_link_libraries(${name} PRIVATE gmock gtest_main ${EXTENSION_NAME}) @@ -71,6 +74,14 @@ function(dd_wrapper_add_test name) gtest_discover_tests(${name} DISCOVERY_MODE PRE_TEST) # Delay test discovery until test execution to avoid running # sanitizer-built executables during build + # Echion headers (vm.h, tasks.h) require PL_DARWIN or PL_LINUX to define proc_ref_t and copy_type. These are set + # PRIVATE on ${EXTENSION_NAME} and don't propagate to test targets, so we replicate the logic here. + if(APPLE) + target_compile_definitions(${name} PRIVATE PL_DARWIN) + else() + target_compile_definitions(${name} PRIVATE PL_LINUX) + endif() + # This is supplemental artifact so make sure to install it in the right place if(INPLACE_LIB_INSTALL_DIR) set(LIB_INSTALL_DIR "${INPLACE_LIB_INSTALL_DIR}") @@ -114,3 +125,17 @@ configure_stack_internal_test(test_sampling_cycle_state) dd_wrapper_add_test(test_alt_stack_ownership test_alt_stack_ownership.cpp) # ThreadAltStack lives in the vendored echion header tree. target_include_directories(test_alt_stack_ownership PRIVATE ../echion) +# test_frame_state_315 validates the PyFrameState renumbering and related frame-internals changes introduced in Python +# 3.15. All test bodies are gated on PY_VERSION_HEX >= 0x030f0000, so building on older versions would produce an empty +# test binary. +if(Python3_VERSION VERSION_GREATER_EQUAL "3.15") + # INSTALL_SUBDIR py315 keeps this binary out of the shared test/ directory. build_base_venvs runs in parallel for + # all Python versions; GitLab merges their artifacts into one directory for downstream jobs. Without isolation the + # py3.15-compiled binary (RPATH -> libpython3.15) would crash when the pytest gtest plugin tried to run it in a + # py3.10 environment. + dd_wrapper_add_test(test_frame_state_315 test_frame_state_315.cpp INSTALL_SUBDIR py315) + # Route copy_memory through echion_fuzz_copy_memory so tests can assert whether the state guard allows execution to + # reach the copy site. + target_compile_definitions(test_frame_state_315 PRIVATE ECHION_FUZZING) +endif() +dd_wrapper_add_test(test_cpython_layout_contracts test_cpython_layout_contracts.cpp) diff --git a/ddtrace/internal/datadog/profiling/stack/test/test_cpython_layout_contracts.cpp b/ddtrace/internal/datadog/profiling/stack/test/test_cpython_layout_contracts.cpp new file mode 100644 index 00000000000..625b46b1e2b --- /dev/null +++ b/ddtrace/internal/datadog/profiling/stack/test/test_cpython_layout_contracts.cpp @@ -0,0 +1,201 @@ +// Compile-time contracts for CPython internal enum values that echion depends on. +// +// Each static_assert fires at *compile time* against the actual CPython headers — +// if CPython renumbers or removes an enum value the build breaks immediately, +// before any test runner is invoked. The matching gtest TEST() wrappers surface +// the same checks as human-readable failures in CI output. +// +// Update these blocks when adding support for a new CPython minor version: +// 1. Add a new versioned block with the new values. +// 2. Adjust the upper-bound on the previous block if values changed. +// 3. Run the build against the new CPython to confirm all assertions pass. +// +// Enums covered: +// _frameowner (pycore_interpframe_structs.h, 3.12+) +// PyFrameState (pycore_frame.h, 3.11+) + +#define PY_SSIZE_T_CLEAN +#define Py_BUILD_CORE +#include + +#include + +#if PY_VERSION_HEX >= 0x030e0000 +// Python 3.14+: frame internals split into separate headers; +// _frameowner is in pycore_interpframe_structs.h (new in 3.14). +#include +#include +#include +#elif PY_VERSION_HEX >= 0x030b0000 +// Python 3.11-3.13: _frameowner enum lives directly in pycore_frame.h. +// pycore_interpframe_structs.h does not exist on these versions. +#include +#endif + +// echion/vm.h defines proc_ref_t, which the stub below requires. +#include + +// Stub: echion's remote-memory callback is referenced at link time via vm.h. +// Always returns failure — no live process attached in unit tests. +extern "C" int +echion_fuzz_copy_memory(proc_ref_t /*proc_ref*/, const void* /*addr*/, ssize_t /*len*/, void* /*buf*/) +{ + return -1; +} + +// ───────────────────────────────────────────────────────────────────────────── +// _frameowner enum (pycore_interpframe_structs.h, introduced in 3.12) +// ───────────────────────────────────────────────────────────────────────────── + +// 3.12 – 3.13: four members, CSTACK=3, no INTERPRETER +#if PY_VERSION_HEX >= 0x030c0000 && PY_VERSION_HEX < 0x030e0000 +static_assert(FRAME_OWNED_BY_THREAD == 0, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_THREAD changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_GENERATOR == 1, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_GENERATOR changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_FRAME_OBJECT == 2, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_FRAME_OBJECT changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_CSTACK == 3, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_CSTACK changed value — update frame.cc owner switch"); + +TEST(FrameOwnerEnum_312_313, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_OWNED_BY_THREAD, 0); + EXPECT_EQ(FRAME_OWNED_BY_GENERATOR, 1); + EXPECT_EQ(FRAME_OWNED_BY_FRAME_OBJECT, 2); + EXPECT_EQ(FRAME_OWNED_BY_CSTACK, 3); +} +#endif // 3.12 – 3.13 + +// 3.14: five members, INTERPRETER added (=3), CSTACK bumped to 4 +#if PY_VERSION_HEX >= 0x030e0000 && PY_VERSION_HEX < 0x030f0000 +static_assert(FRAME_OWNED_BY_THREAD == 0, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_THREAD changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_GENERATOR == 1, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_GENERATOR changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_FRAME_OBJECT == 2, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_FRAME_OBJECT changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_INTERPRETER == 3, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_INTERPRETER changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_CSTACK == 4, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_CSTACK changed value — update frame.cc owner switch"); + +TEST(FrameOwnerEnum_314, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_OWNED_BY_THREAD, 0); + EXPECT_EQ(FRAME_OWNED_BY_GENERATOR, 1); + EXPECT_EQ(FRAME_OWNED_BY_FRAME_OBJECT, 2); + EXPECT_EQ(FRAME_OWNED_BY_INTERPRETER, 3); + EXPECT_EQ(FRAME_OWNED_BY_CSTACK, 4); +} +#endif // 3.14 + +// 3.15+: FRAME_OWNED_BY_CSTACK removed +#if PY_VERSION_HEX >= 0x030f0000 +static_assert(FRAME_OWNED_BY_THREAD == 0, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_THREAD changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_GENERATOR == 1, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_GENERATOR changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_FRAME_OBJECT == 2, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_FRAME_OBJECT changed value — update frame.cc owner switch"); +static_assert(FRAME_OWNED_BY_INTERPRETER == 3, + "TODO(py-315): _frameowner::FRAME_OWNED_BY_INTERPRETER changed value — update frame.cc owner switch"); +// FRAME_OWNED_BY_CSTACK intentionally not listed — it was removed in 3.15. +// If this file compiles without error, CPython has not re-introduced it. + +TEST(FrameOwnerEnum_315, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_OWNED_BY_THREAD, 0); + EXPECT_EQ(FRAME_OWNED_BY_GENERATOR, 1); + EXPECT_EQ(FRAME_OWNED_BY_FRAME_OBJECT, 2); + EXPECT_EQ(FRAME_OWNED_BY_INTERPRETER, 3); +} +#endif // 3.15+ + +// ───────────────────────────────────────────────────────────────────────────── +// PyFrameState / gi_frame_state (pycore_frame.h, introduced in 3.11) +// ───────────────────────────────────────────────────────────────────────────── + +// 3.11 – 3.12: negative-valued range, no FRAME_SUSPENDED_YIELD_FROM yet. +// FRAME_SUSPENDED_YIELD_FROM was introduced in 3.13 (CPython gh-104210), which +// also shifted FRAME_CREATED and FRAME_SUSPENDED one slot more negative. +#if PY_VERSION_HEX >= 0x030b0000 && PY_VERSION_HEX < 0x030d0000 +static_assert(FRAME_CREATED == -2, + "TODO(py-315): PyFrameState::FRAME_CREATED changed value — update tasks.h PyGen_yf and tasks.cc"); +static_assert(FRAME_SUSPENDED == -1, "TODO(py-315): PyFrameState::FRAME_SUSPENDED changed value"); +static_assert(FRAME_EXECUTING == 0, + "TODO(py-315): PyFrameState::FRAME_EXECUTING changed value — update tasks.cc gen_is_running check"); +// FRAME_COMPLETED == 1 is not used by echion directly; omitted intentionally. +static_assert(FRAME_CLEARED == 4, + "TODO(py-315): PyFrameState::FRAME_CLEARED changed value — update tasks.cc gi_frame_state check"); + +TEST(PyFrameStateEnum_311_312, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_CREATED, -2); + EXPECT_EQ(FRAME_SUSPENDED, -1); + EXPECT_EQ(FRAME_EXECUTING, 0); + EXPECT_EQ(FRAME_CLEARED, 4); +} +#endif // 3.11 – 3.12 + +// 3.13 – 3.14: negative-valued range, FRAME_SUSPENDED_YIELD_FROM added (-1), +// pushing FRAME_CREATED to -3 and FRAME_SUSPENDED to -2. +#if PY_VERSION_HEX >= 0x030d0000 && PY_VERSION_HEX < 0x030f0000 +static_assert(FRAME_CREATED == -3, + "TODO(py-315): PyFrameState::FRAME_CREATED changed value — update tasks.h PyGen_yf and tasks.cc"); +static_assert(FRAME_SUSPENDED == -2, "TODO(py-315): PyFrameState::FRAME_SUSPENDED changed value"); +static_assert(FRAME_SUSPENDED_YIELD_FROM == -1, + "TODO(py-315): PyFrameState::FRAME_SUSPENDED_YIELD_FROM changed value — update tasks.h PyGen_yf"); +static_assert(FRAME_EXECUTING == 0, + "TODO(py-315): PyFrameState::FRAME_EXECUTING changed value — update tasks.cc gen_is_running check"); +// FRAME_COMPLETED == 1 is not used by echion directly; omitted intentionally. +static_assert(FRAME_CLEARED == 4, + "TODO(py-315): PyFrameState::FRAME_CLEARED changed value — update tasks.cc gi_frame_state check"); + +TEST(PyFrameStateEnum_313_314, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_CREATED, -3); + EXPECT_EQ(FRAME_SUSPENDED, -2); + EXPECT_EQ(FRAME_SUSPENDED_YIELD_FROM, -1); + EXPECT_EQ(FRAME_EXECUTING, 0); + EXPECT_EQ(FRAME_CLEARED, 4); +} +#endif // 3.13 – 3.14 + +// 3.15+: all values renumbered, COMPLETED removed +#if PY_VERSION_HEX >= 0x030f0000 +static_assert(FRAME_CREATED == 0, + "TODO(py-315): PyFrameState::FRAME_CREATED changed value — update tasks.h PyGen_yf and tasks.cc"); +static_assert(FRAME_SUSPENDED == 1, "TODO(py-315): PyFrameState::FRAME_SUSPENDED changed value"); +static_assert(FRAME_SUSPENDED_YIELD_FROM == 2, + "TODO(py-315): PyFrameState::FRAME_SUSPENDED_YIELD_FROM changed value — update tasks.h PyGen_yf"); +// value 3 is FRAME_SUSPENDED_YIELD_FROM_LOCKED in free-threaded builds (see below) +static_assert(FRAME_EXECUTING == 4, + "TODO(py-315): PyFrameState::FRAME_EXECUTING changed value — update tasks.cc gen_is_running check"); +static_assert(FRAME_CLEARED == 5, + "TODO(py-315): PyFrameState::FRAME_CLEARED changed value — update tasks.cc gi_frame_state check"); +// FRAME_COMPLETED intentionally not listed — it was removed in 3.15. + +#ifdef Py_GIL_DISABLED +static_assert( + FRAME_SUSPENDED_YIELD_FROM_LOCKED == 3, + "TODO(py-315): FRAME_SUSPENDED_YIELD_FROM_LOCKED changed value — update tasks.h PyGen_yf (Py_GIL_DISABLED)"); +#endif + +TEST(PyFrameStateEnum_315, ValuesMatchExpected) +{ + EXPECT_EQ(FRAME_CREATED, 0); + EXPECT_EQ(FRAME_SUSPENDED, 1); + EXPECT_EQ(FRAME_SUSPENDED_YIELD_FROM, 2); + EXPECT_EQ(FRAME_EXECUTING, 4); + EXPECT_EQ(FRAME_CLEARED, 5); +} + +#ifdef Py_GIL_DISABLED +TEST(PyFrameStateEnum_315_NoGIL, LockedYieldFromValueMatchesExpected) +{ + EXPECT_EQ(FRAME_SUSPENDED_YIELD_FROM_LOCKED, 3); +} +#endif + +#endif // 3.15+ diff --git a/ddtrace/internal/datadog/profiling/stack/test/test_frame_state_315.cpp b/ddtrace/internal/datadog/profiling/stack/test/test_frame_state_315.cpp new file mode 100644 index 00000000000..bb356d5fc8a --- /dev/null +++ b/ddtrace/internal/datadog/profiling/stack/test/test_frame_state_315.cpp @@ -0,0 +1,146 @@ +// Unit tests for Python 3.15 frame-state guard changes. +// +// Covered: +// 1. Static assertions on renumbered PyFrameState enum values (3.15+). +// 2. PyGen_yf returns nullptr for FRAME_SUSPENDED_YIELD_FROM_LOCKED in GIL builds (3.15+). +// 3. PyGen_yf enters the body for FRAME_SUSPENDED_YIELD_FROM even after the 3.15 guard change. +// 4. PyGen_yf returns nullptr for all non-suspended states (3.15+). +// +// Memory stub: copy_type/copy_generic call echion_fuzz_copy_memory. We define it here to +// always return failure (-1), which is the correct outcome when no real Python process is +// attached. All code paths that reach a copy_type call will return nullptr safely. + +#define PY_SSIZE_T_CLEAN +#define Py_BUILD_CORE +#include + +#include + +#include +#include + +#if PY_VERSION_HEX >= 0x030e0000 +#include +#include +#include +#include +#include +#endif + +#include +#include +#include + +// Counter tracking how many times copy_memory was invoked. Reset before each +// PyGen_yf call so tests can assert whether the state guard allowed execution +// to reach the copy site (>0) or filtered it out first (0). +// Must be declared before echion headers use ECHION_FUZZING to route copy_memory +// through this stub; atomic so future parallel-test runs stay race-free. +static std::atomic g_copy_attempts{ 0 }; + +extern "C" int +echion_fuzz_copy_memory(proc_ref_t /*proc_ref*/, const void* /*addr*/, ssize_t /*len*/, void* /*buf*/) +{ + g_copy_attempts.fetch_add(1, std::memory_order_relaxed); + return -1; // always fail — no live process attached +} + +// ───────────────────────────────────────────────────────────────────────────── +// 1. Compile-time enum value assertions (3.15+ only) +// ───────────────────────────────────────────────────────────────────────────── + +#if PY_VERSION_HEX >= 0x030f0000 + +// PyFrameState was renumbered in 3.15. Verify our understanding matches reality so +// that any future CPython change is caught immediately at compile time. +static_assert(FRAME_CREATED == 0, "FRAME_CREATED should be 0 in Python 3.15"); +static_assert(FRAME_SUSPENDED == 1, "FRAME_SUSPENDED should be 1 in Python 3.15"); +static_assert(FRAME_SUSPENDED_YIELD_FROM == 2, "FRAME_SUSPENDED_YIELD_FROM should be 2 in Python 3.15"); +static_assert(FRAME_EXECUTING == 4, "FRAME_EXECUTING should be 4 in Python 3.15"); +static_assert(FRAME_CLEARED == 5, "FRAME_CLEARED should be 5 in Python 3.15"); + +#ifdef Py_GIL_DISABLED +// FRAME_SUSPENDED_YIELD_FROM_LOCKED only exists when building against a free-threaded Python. +static_assert(FRAME_SUSPENDED_YIELD_FROM_LOCKED == 3, "FRAME_SUSPENDED_YIELD_FROM_LOCKED should be 3 in Python 3.15"); +#endif // Py_GIL_DISABLED + +TEST(PyFrameState315, EnumValuesMatchExpected) +{ + // Runtime counterpart of the static_asserts above — provides a readable failure + // message in the test output if run against an unexpected Python build. + EXPECT_EQ(FRAME_CREATED, 0); + EXPECT_EQ(FRAME_SUSPENDED, 1); + EXPECT_EQ(FRAME_SUSPENDED_YIELD_FROM, 2); + EXPECT_EQ(FRAME_EXECUTING, 4); + EXPECT_EQ(FRAME_CLEARED, 5); +} + +// ───────────────────────────────────────────────────────────────────────────── +// 2. PyGen_yf state-check tests (3.15+) +// +// PyGenObject::gi_frame_state is an int (signed). We set only that field; all +// other fields are zero-initialised. We pass nullptr as frame_addr so that if the +// state check passes, copy_type will immediately fail and return nullptr — which +// means any test that expects nullptr is still correct regardless of whether the +// state check or the copy fails first. +// ───────────────────────────────────────────────────────────────────────────── + +static PyGenObject +make_fake_gen(int frame_state) +{ + PyGenObject gen{}; + gen.gi_frame_state = frame_state; + return gen; +} + +#ifndef Py_GIL_DISABLED + +TEST(PyGenYf315GilBuild, LockedStateIgnored) +{ + // FRAME_SUSPENDED_YIELD_FROM_LOCKED (value 3) must NOT be treated as a + // suspended-yield-from state in GIL builds. PyGen_yf should return nullptr + // immediately from the state guard without attempting any memory read. + g_copy_attempts.store(0, std::memory_order_relaxed); + auto gen = make_fake_gen(3 /* FRAME_SUSPENDED_YIELD_FROM_LOCKED value */); + PyObject* result = PyGen_yf(&gen, nullptr); + EXPECT_EQ(result, nullptr); + EXPECT_EQ(g_copy_attempts.load(std::memory_order_relaxed), 0) + << "state guard must filter FRAME_SUSPENDED_YIELD_FROM_LOCKED without any copy attempt"; +} + +TEST(PyGenYf315GilBuild, SuspendedYieldFromEntersBody) +{ + // FRAME_SUSPENDED_YIELD_FROM must still be recognised as a suspended state. + // The state guard passes, execution enters the body, and copy_type(nullptr, frame) + // immediately fails — confirming the guard did NOT filter out this state. + g_copy_attempts.store(0, std::memory_order_relaxed); + auto gen = make_fake_gen(FRAME_SUSPENDED_YIELD_FROM); + PyObject* result = PyGen_yf(&gen, nullptr); + EXPECT_EQ(result, nullptr); // copy_type fails on nullptr frame_addr + EXPECT_GT(g_copy_attempts.load(std::memory_order_relaxed), 0) + << "FRAME_SUSPENDED_YIELD_FROM must pass the state guard and attempt a copy"; +} + +#endif // !Py_GIL_DISABLED + +// Parametrised: non-suspended states must all return nullptr immediately. +class PyGenYf315OtherStates : public ::testing::TestWithParam +{}; + +TEST_P(PyGenYf315OtherStates, ReturnsNull) +{ + g_copy_attempts.store(0, std::memory_order_relaxed); + auto gen = make_fake_gen(GetParam()); + EXPECT_EQ(PyGen_yf(&gen, nullptr), nullptr); + EXPECT_EQ(g_copy_attempts.load(std::memory_order_relaxed), 0) + << "non-suspended states must be filtered by the state guard without any copy attempt"; +} + +INSTANTIATE_TEST_SUITE_P(NonSuspendedStates, + PyGenYf315OtherStates, + ::testing::Values(FRAME_CREATED, // 0 + FRAME_EXECUTING, // 4 + FRAME_CLEARED // 5 + )); + +#endif // PY_VERSION_HEX >= 0x030f0000 From 9d8528db4680ae5821fec691a9bc9b79e81423d3 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Fri, 21 Aug 2026 07:51:19 +0300 Subject: [PATCH 03/14] chore(profiling): update Python profiling collectors for py3.15 # Conflicts: # ddtrace/internal/monitoring.py # ddtrace/internal/wrapping/asyncs.py --- ddtrace/internal/monitoring.py | 15 +++- ddtrace/profiling/_asyncio.py | 47 ++++++------ ddtrace/profiling/collector/asyncio.py | 77 +++++++++++--------- ddtrace/profiling/collector/exception.py | 14 +++- ddtrace/profiling/collector/stack.py | 11 ++- ddtrace/profiling/collector/threading.py | 92 +++++++++++++----------- tests/profiling/test_scheduler.py | 5 +- 7 files changed, 162 insertions(+), 99 deletions(-) diff --git a/ddtrace/internal/monitoring.py b/ddtrace/internal/monitoring.py index f63994cd711..e7ae535c81a 100644 --- a/ddtrace/internal/monitoring.py +++ b/ddtrace/internal/monitoring.py @@ -142,6 +142,7 @@ def _events_for_handler(handler: MonitoringEventHandler) -> int: return events + class _Entry(NamedTuple): handler: MonitoringEventHandler events: int # pre-computed from _events_for_handler @@ -274,7 +275,19 @@ def _on_py_line(code: CodeType, line_number: int) -> Optional[object]: def _set_local_events(tool_id: int, code: CodeType, events: int) -> None: - sys.monitoring.set_local_events(tool_id, code, events) + # TODO(py-315): Pre-release Python 3.15 builds may reject PY_UNWIND + # as a local event. Fall back without it when the full set is invalid; + # PY_UNWIND is still registered as a global callback via _setup() so + # exception handling degrades gracefully rather than crashing. + try: + sys.monitoring.set_local_events(tool_id, code, events) + except ValueError: + fallback = events & ~_E.PY_UNWIND + if fallback != events: + sys.monitoring.set_local_events(tool_id, code, fallback) + else: + raise + def _rearm_local_events(tool_id: int, code: CodeType, events: int) -> None: diff --git a/ddtrace/profiling/_asyncio.py b/ddtrace/profiling/_asyncio.py index 133dab9ddb5..259bae9eaeb 100644 --- a/ddtrace/profiling/_asyncio.py +++ b/ddtrace/profiling/_asyncio.py @@ -119,19 +119,21 @@ def _( @partial(wrap, sys.modules["asyncio"].tasks._GatheringFuture.__init__) def _(f: typing.Callable[..., None], args: tuple[typing.Any, ...], kwargs: dict[str, typing.Any]) -> None: + f(*args, **kwargs) + children = get_argument_value(args, kwargs, 1, "children") + assert children is not None # nosec: assert is used for typing + + # TODO(py-315): current_task() raises RuntimeError on Python 3.15+ when there + # is no running event loop (e.g. asyncio.gather() called outside an async + # context to build a coroutine for later scheduling). In that case there is + # no parent task to link from, so we skip link_tasks entirely. try: - return f(*args, **kwargs) - finally: - children: list[aio.Future[typing.Any]] = typing.cast( - "list[aio.Future[typing.Any]]", get_argument_value(args, kwargs, 1, "children") - ) - assert children is not None # nosec: assert is used for typing - - if globals()["get_running_loop"]() is not None: - parent: typing.Optional[aio.Task[typing.Any]] = globals()["current_task"]() - if parent is not None: - for child in children: - stack.link_tasks(parent, child) + parent = globals()["current_task"]() + except RuntimeError: + return + if parent is not None: + for child in children: + stack.link_tasks(parent, child) @partial(wrap, sys.modules["asyncio"].tasks._wait) def _( @@ -139,15 +141,20 @@ def _( args: tuple[typing.Any, ...], kwargs: dict[str, typing.Any], ) -> typing.Any: + result = f(*args, **kwargs) + futures = typing.cast(set["aio.Future[typing.Any]"], get_argument_value(args, kwargs, 0, "fs")) + + # TODO(py-315): same guard as the _GatheringFuture wrapper above — _wait may + # also be invoked outside a running loop. Skip link_tasks when current_task() + # raises. try: - return f(*args, **kwargs) - finally: - futures = typing.cast("set[aio.Future[typing.Any]]", get_argument_value(args, kwargs, 0, "fs")) - - if globals()["get_running_loop"]() is not None: - parent = typing.cast("aio.Task[typing.Any]", globals()["current_task"]()) - for future in futures: - stack.link_tasks(parent, future) + parent = typing.cast("aio.Task[typing.Any]", globals()["current_task"]()) + except RuntimeError: + return result + if parent is not None: + for future in futures: + stack.link_tasks(parent, future) + return result @partial(wrap, sys.modules["asyncio"].tasks.as_completed) def _( diff --git a/ddtrace/profiling/collector/asyncio.py b/ddtrace/profiling/collector/asyncio.py index d8afdf93aeb..47933c6bcec 100644 --- a/ddtrace/profiling/collector/asyncio.py +++ b/ddtrace/profiling/collector/asyncio.py @@ -3,52 +3,63 @@ import asyncio from types import ModuleType -from . import _lock +try: + from . import _lock -class _ProfiledAsyncioLock(_lock._ProfiledLock): - pass + class _ProfiledAsyncioLock(_lock._ProfiledLock): + pass + class _ProfiledAsyncioSemaphore(_lock._ProfiledLock): + pass -class _ProfiledAsyncioSemaphore(_lock._ProfiledLock): - pass + class _ProfiledAsyncioBoundedSemaphore(_lock._ProfiledLock): + pass + class _ProfiledAsyncioCondition(_lock._ProfiledLock): + pass -class _ProfiledAsyncioBoundedSemaphore(_lock._ProfiledLock): - pass + class AsyncioLockCollector(_lock.LockCollector): + """Record asyncio.Lock usage.""" + PROFILED_LOCK_CLASS: type[_ProfiledAsyncioLock] = _ProfiledAsyncioLock + MODULE: ModuleType = asyncio + PATCHED_LOCK_NAME: str = "Lock" -class _ProfiledAsyncioCondition(_lock._ProfiledLock): - pass + class AsyncioSemaphoreCollector(_lock.LockCollector): + """Record asyncio.Semaphore usage.""" + PROFILED_LOCK_CLASS: type[_ProfiledAsyncioSemaphore] = _ProfiledAsyncioSemaphore + MODULE: ModuleType = asyncio + PATCHED_LOCK_NAME: str = "Semaphore" -class AsyncioLockCollector(_lock.LockCollector): - """Record asyncio.Lock usage.""" + class AsyncioBoundedSemaphoreCollector(_lock.LockCollector): + """Record asyncio.BoundedSemaphore usage.""" - PROFILED_LOCK_CLASS: type[_ProfiledAsyncioLock] = _ProfiledAsyncioLock - MODULE: ModuleType = asyncio - PATCHED_LOCK_NAME: str = "Lock" + PROFILED_LOCK_CLASS: type[_ProfiledAsyncioBoundedSemaphore] = _ProfiledAsyncioBoundedSemaphore + MODULE: ModuleType = asyncio + PATCHED_LOCK_NAME: str = "BoundedSemaphore" + class AsyncioConditionCollector(_lock.LockCollector): + """Record asyncio.Condition usage.""" -class AsyncioSemaphoreCollector(_lock.LockCollector): - """Record asyncio.Semaphore usage.""" + PROFILED_LOCK_CLASS: type[_ProfiledAsyncioCondition] = _ProfiledAsyncioCondition + MODULE: ModuleType = asyncio + PATCHED_LOCK_NAME: str = "Condition" - PROFILED_LOCK_CLASS: type[_ProfiledAsyncioSemaphore] = _ProfiledAsyncioSemaphore - MODULE: ModuleType = asyncio - PATCHED_LOCK_NAME: str = "Semaphore" +except ImportError: + # TODO(py-315): _lock is a Cython extension that is not compiled for all Python + # versions (e.g. Python 3.15 before the manylinux image carries it). When it + # is absent the asyncio lock collectors are unavailable. Defining stubs that + # raise CollectorUnavailable lets profiler.py discover and gracefully skip them + # rather than failing at import time. + from ddtrace.profiling.collector import Collector as _Collector + from ddtrace.profiling.collector import CollectorUnavailable as _CollectorUnavailable + class AsyncioLockCollector(_Collector): # type: ignore[no-redef] + def start(self) -> None: + raise _CollectorUnavailable -class AsyncioBoundedSemaphoreCollector(_lock.LockCollector): - """Record asyncio.BoundedSemaphore usage.""" - - PROFILED_LOCK_CLASS: type[_ProfiledAsyncioBoundedSemaphore] = _ProfiledAsyncioBoundedSemaphore - MODULE: ModuleType = asyncio - PATCHED_LOCK_NAME: str = "BoundedSemaphore" - - -class AsyncioConditionCollector(_lock.LockCollector): - """Record asyncio.Condition usage.""" - - PROFILED_LOCK_CLASS: type[_ProfiledAsyncioCondition] = _ProfiledAsyncioCondition - MODULE: ModuleType = asyncio - PATCHED_LOCK_NAME: str = "Condition" + AsyncioSemaphoreCollector = AsyncioLockCollector # type: ignore[assignment,misc] + AsyncioBoundedSemaphoreCollector = AsyncioLockCollector # type: ignore[assignment,misc] + AsyncioConditionCollector = AsyncioLockCollector # type: ignore[assignment,misc] diff --git a/ddtrace/profiling/collector/exception.py b/ddtrace/profiling/collector/exception.py index af0877d07d7..851974b84b4 100644 --- a/ddtrace/profiling/collector/exception.py +++ b/ddtrace/profiling/collector/exception.py @@ -1,4 +1,16 @@ -from ddtrace.profiling.collector._exception import ExceptionCollector +try: + from ddtrace.profiling.collector._exception import ExceptionCollector +except ImportError: + # TODO(py-315): _exception is a Cython extension not compiled for all Python + # versions (e.g. Python 3.15 before the manylinux image carries it). Define + # a stub so profiler.py can import this module and skip the collector via + # CollectorUnavailable rather than failing at import time. + from ddtrace.profiling.collector import Collector as _Collector + from ddtrace.profiling.collector import CollectorUnavailable as _CollectorUnavailable + + class ExceptionCollector(_Collector): # type: ignore[no-redef] + def start(self) -> None: + raise _CollectorUnavailable __all__ = ["ExceptionCollector"] diff --git a/ddtrace/profiling/collector/stack.py b/ddtrace/profiling/collector/stack.py index 406e875eef1..c72ff8305fd 100644 --- a/ddtrace/profiling/collector/stack.py +++ b/ddtrace/profiling/collector/stack.py @@ -13,7 +13,16 @@ from ddtrace.internal.datadog.profiling import stack from ddtrace.internal.settings.profiling import config from ddtrace.profiling import collector -from ddtrace.profiling.collector import _task + + +try: + from ddtrace.profiling.collector import _task +except ImportError: + # TODO(py-315): _task is a Cython extension not compiled for all Python versions. + # Provide a no-op stub so StackCollector can be imported on Python 3.15. + import types as _types + + _task = _types.SimpleNamespace(initialize_gevent_support=lambda: None) # type: ignore[assignment] from ddtrace.profiling.collector import threading from ddtrace.trace import Tracer diff --git a/ddtrace/profiling/collector/threading.py b/ddtrace/profiling/collector/threading.py index 2d15d124b85..2ec931f1a9e 100644 --- a/ddtrace/profiling/collector/threading.py +++ b/ddtrace/profiling/collector/threading.py @@ -6,67 +6,75 @@ from ddtrace.internal.datadog.profiling import stack from ddtrace.internal.settings.profiling import config -from . import _lock +try: + from . import _lock -class _ProfiledThreadingLock(_lock._ProfiledLock): - pass + class _ProfiledThreadingLock(_lock._ProfiledLock): + pass + class _ProfiledThreadingRLock(_lock._ProfiledLock): + pass -class _ProfiledThreadingRLock(_lock._ProfiledLock): - pass + class _ProfiledThreadingSemaphore(_lock._ProfiledLock): + pass + class _ProfiledThreadingBoundedSemaphore(_lock._ProfiledLock): + pass -class _ProfiledThreadingSemaphore(_lock._ProfiledLock): - pass + class _ProfiledThreadingCondition(_lock._ProfiledLock): + pass + class ThreadingLockCollector(_lock.LockCollector): + """Record threading.Lock usage.""" -class _ProfiledThreadingBoundedSemaphore(_lock._ProfiledLock): - pass + PROFILED_LOCK_CLASS: type[_ProfiledThreadingLock] = _ProfiledThreadingLock + MODULE: ModuleType = threading + PATCHED_LOCK_NAME: str = "Lock" + class ThreadingRLockCollector(_lock.LockCollector): + """Record threading.RLock usage.""" -class _ProfiledThreadingCondition(_lock._ProfiledLock): - pass + PROFILED_LOCK_CLASS: type[_ProfiledThreadingRLock] = _ProfiledThreadingRLock + MODULE: ModuleType = threading + PATCHED_LOCK_NAME: str = "RLock" + class ThreadingSemaphoreCollector(_lock.LockCollector): + """Record threading.Semaphore usage.""" -class ThreadingLockCollector(_lock.LockCollector): - """Record threading.Lock usage.""" + PROFILED_LOCK_CLASS: type[_ProfiledThreadingSemaphore] = _ProfiledThreadingSemaphore + MODULE: ModuleType = threading + PATCHED_LOCK_NAME: str = "Semaphore" - PROFILED_LOCK_CLASS: type[_ProfiledThreadingLock] = _ProfiledThreadingLock - MODULE: ModuleType = threading - PATCHED_LOCK_NAME: str = "Lock" + class ThreadingBoundedSemaphoreCollector(_lock.LockCollector): + """Record threading.BoundedSemaphore usage.""" + PROFILED_LOCK_CLASS: type[_ProfiledThreadingBoundedSemaphore] = _ProfiledThreadingBoundedSemaphore + MODULE: ModuleType = threading + PATCHED_LOCK_NAME: str = "BoundedSemaphore" -class ThreadingRLockCollector(_lock.LockCollector): - """Record threading.RLock usage.""" + class ThreadingConditionCollector(_lock.LockCollector): + """Record threading.Condition usage.""" - PROFILED_LOCK_CLASS: type[_ProfiledThreadingRLock] = _ProfiledThreadingRLock - MODULE: ModuleType = threading - PATCHED_LOCK_NAME: str = "RLock" + PROFILED_LOCK_CLASS: type[_ProfiledThreadingCondition] = _ProfiledThreadingCondition + MODULE: ModuleType = threading + PATCHED_LOCK_NAME: str = "Condition" +except ImportError: + # TODO(py-315): _lock is a Cython extension not compiled for all Python versions + # (e.g. Python 3.15 before the manylinux image carries it). Stubs raise + # CollectorUnavailable so profiler.py skips them gracefully. + from ddtrace.profiling.collector import Collector as _Collector + from ddtrace.profiling.collector import CollectorUnavailable as _CollectorUnavailable -class ThreadingSemaphoreCollector(_lock.LockCollector): - """Record threading.Semaphore usage.""" + class ThreadingLockCollector(_Collector): # type: ignore[no-redef] + def start(self) -> None: + raise _CollectorUnavailable - PROFILED_LOCK_CLASS: type[_ProfiledThreadingSemaphore] = _ProfiledThreadingSemaphore - MODULE: ModuleType = threading - PATCHED_LOCK_NAME: str = "Semaphore" - - -class ThreadingBoundedSemaphoreCollector(_lock.LockCollector): - """Record threading.BoundedSemaphore usage.""" - - PROFILED_LOCK_CLASS: type[_ProfiledThreadingBoundedSemaphore] = _ProfiledThreadingBoundedSemaphore - MODULE: ModuleType = threading - PATCHED_LOCK_NAME: str = "BoundedSemaphore" - - -class ThreadingConditionCollector(_lock.LockCollector): - """Record threading.Condition usage.""" - - PROFILED_LOCK_CLASS: type[_ProfiledThreadingCondition] = _ProfiledThreadingCondition - MODULE: ModuleType = threading - PATCHED_LOCK_NAME: str = "Condition" + ThreadingRLockCollector = ThreadingLockCollector # type: ignore[assignment,misc] + ThreadingSemaphoreCollector = ThreadingLockCollector # type: ignore[assignment,misc] + ThreadingBoundedSemaphoreCollector = ThreadingLockCollector # type: ignore[assignment,misc] + ThreadingConditionCollector = ThreadingLockCollector # type: ignore[assignment,misc] # Also patch threading.Thread so echion can track thread lifetimes diff --git a/tests/profiling/test_scheduler.py b/tests/profiling/test_scheduler.py index 75a0c77c098..9cbb2e2bda3 100644 --- a/tests/profiling/test_scheduler.py +++ b/tests/profiling/test_scheduler.py @@ -34,7 +34,10 @@ def call_me(): raise Exception("LOL") s = scheduler.Scheduler(before_flush=call_me) - s.flush() + # Patch ddup.upload so the test only checks scheduler logging behaviour and + # doesn't attempt a real upload (which would log a writer error and pollute caplog). + with mock.patch("ddtrace.profiling.scheduler.ddup.upload"): + s.flush() assert caplog.record_tuples == [ (("ddtrace.profiling.scheduler", logging.ERROR, "Scheduler before_flush hook failed")) ] From 6f9a44c2560a2f4cb8cea9db178ba221903f9cfb Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Thu, 23 Jul 2026 10:28:48 -0400 Subject: [PATCH 04/14] typing: annotate profiling collectors for py3.15 --- ddtrace/internal/monitoring.py | 2 +- ddtrace/profiling/_asyncio.py | 16 +++++++++++----- ddtrace/profiling/collector/stack.py | 5 ++++- tests/profiling/test_scheduler.py | 6 ++++-- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/ddtrace/internal/monitoring.py b/ddtrace/internal/monitoring.py index e7ae535c81a..10e7ee87525 100644 --- a/ddtrace/internal/monitoring.py +++ b/ddtrace/internal/monitoring.py @@ -282,7 +282,7 @@ def _set_local_events(tool_id: int, code: CodeType, events: int) -> None: try: sys.monitoring.set_local_events(tool_id, code, events) except ValueError: - fallback = events & ~_E.PY_UNWIND + fallback: int = events & ~_E.PY_UNWIND if fallback != events: sys.monitoring.set_local_events(tool_id, code, fallback) else: diff --git a/ddtrace/profiling/_asyncio.py b/ddtrace/profiling/_asyncio.py index 259bae9eaeb..9ac02e80dec 100644 --- a/ddtrace/profiling/_asyncio.py +++ b/ddtrace/profiling/_asyncio.py @@ -120,7 +120,9 @@ def _( @partial(wrap, sys.modules["asyncio"].tasks._GatheringFuture.__init__) def _(f: typing.Callable[..., None], args: tuple[typing.Any, ...], kwargs: dict[str, typing.Any]) -> None: f(*args, **kwargs) - children = get_argument_value(args, kwargs, 1, "children") + children: list[aio.Future[typing.Any]] = typing.cast( + "list[aio.Future[typing.Any]]", get_argument_value(args, kwargs, 1, "children") + ) assert children is not None # nosec: assert is used for typing # TODO(py-315): current_task() raises RuntimeError on Python 3.15+ when there @@ -128,7 +130,7 @@ def _(f: typing.Callable[..., None], args: tuple[typing.Any, ...], kwargs: dict[ # context to build a coroutine for later scheduling). In that case there is # no parent task to link from, so we skip link_tasks entirely. try: - parent = globals()["current_task"]() + parent: typing.Optional[aio.Task[typing.Any]] = globals()["current_task"]() except RuntimeError: return if parent is not None: @@ -141,14 +143,18 @@ def _( args: tuple[typing.Any, ...], kwargs: dict[str, typing.Any], ) -> typing.Any: - result = f(*args, **kwargs) - futures = typing.cast(set["aio.Future[typing.Any]"], get_argument_value(args, kwargs, 0, "fs")) + result: tuple[set[aio.Future[typing.Any]], set[aio.Future[typing.Any]]] = f(*args, **kwargs) + futures: set[aio.Future[typing.Any]] = typing.cast( + "set[aio.Future[typing.Any]]", get_argument_value(args, kwargs, 0, "fs") + ) # TODO(py-315): same guard as the _GatheringFuture wrapper above — _wait may # also be invoked outside a running loop. Skip link_tasks when current_task() # raises. try: - parent = typing.cast("aio.Task[typing.Any]", globals()["current_task"]()) + parent: typing.Optional[aio.Task[typing.Any]] = typing.cast( + "aio.Task[typing.Any]", globals()["current_task"]() + ) except RuntimeError: return result if parent is not None: diff --git a/ddtrace/profiling/collector/stack.py b/ddtrace/profiling/collector/stack.py index c72ff8305fd..bf2c45ef4ef 100644 --- a/ddtrace/profiling/collector/stack.py +++ b/ddtrace/profiling/collector/stack.py @@ -22,7 +22,10 @@ # Provide a no-op stub so StackCollector can be imported on Python 3.15. import types as _types - _task = _types.SimpleNamespace(initialize_gevent_support=lambda: None) # type: ignore[assignment] + def _initialize_gevent_support() -> None: + return None + + _task = _types.SimpleNamespace(initialize_gevent_support=_initialize_gevent_support) # type: ignore[assignment] from ddtrace.profiling.collector import threading from ddtrace.trace import Tracer diff --git a/tests/profiling/test_scheduler.py b/tests/profiling/test_scheduler.py index 9cbb2e2bda3..53e1406bbae 100644 --- a/tests/profiling/test_scheduler.py +++ b/tests/profiling/test_scheduler.py @@ -2,6 +2,8 @@ import logging from unittest import mock +import pytest + from ddtrace.profiling import scheduler @@ -29,8 +31,8 @@ def call_me(): assert x["OK"] -def test_before_flush_failure(caplog): - def call_me(): +def test_before_flush_failure(caplog: pytest.LogCaptureFixture) -> None: + def call_me() -> None: raise Exception("LOL") s = scheduler.Scheduler(before_flush=call_me) From 10fd21c0ec18e46641f074e7758e861b3bd78f8e Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Fri, 21 Aug 2026 07:52:05 +0300 Subject: [PATCH 05/14] ci(profiling): wire py3.15 into build matrix, riotfile, and CI Profiling-native py3.15 job, setup.py guards, and crashtracker 3.15 opt-in. Rebased onto the #17849 split stack (PR 17624). # Conflicts: # riotfile.py # setup.py --- .../workflows/generate-package-versions.yml | 5 +++ .gitlab-ci.yml | 16 +++++++ .gitlab/templates/build-base-venvs.yml | 2 + .riot/requirements/1c6cb02.txt | 34 ++++++++++++++ .riot/requirements/222bcd0.txt | 45 +++++++++++++++++++ .riot/requirements/95077af.txt | 33 ++++++++++++++ .riot/requirements/e26245b.txt | 37 +++++++++++++++ riotfile.py | 7 +-- scripts/requirements_to_csv.py | 4 +- setup.py | 11 ++++- 10 files changed, 188 insertions(+), 6 deletions(-) create mode 100644 .riot/requirements/1c6cb02.txt create mode 100644 .riot/requirements/222bcd0.txt create mode 100644 .riot/requirements/95077af.txt create mode 100644 .riot/requirements/e26245b.txt diff --git a/.github/workflows/generate-package-versions.yml b/.github/workflows/generate-package-versions.yml index 955d1552cb4..e742ecfb564 100644 --- a/.github/workflows/generate-package-versions.yml +++ b/.github/workflows/generate-package-versions.yml @@ -53,6 +53,11 @@ jobs: with: python-version: "3.14" + - name: Setup Python 3.15 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.15" + - name: Set up QEMU uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7620cfeedf2..cc047127451 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -703,6 +703,22 @@ profiling_native: - PYTHON_VERSION: ["3.12", "3.14"] SANITIZER: ["valgrind"] +# AIDEV-TODO(py315): fold 3.15 back into the profiling_native matrix once the +# dd/images/dd-trace-py/profiling_native base image installs a Python 3.15 +# pyenv version. Today it only carries 3.9–3.14, so PYENV_VERSION=3.15 fails +# with "pyenv: version `3.15' is not installed". allow_failure keeps the +# pipeline green while the upstream image catches up; delete this job and +# re-add "3.15" to the PYTHON_VERSION arrays above once the image is updated. +profiling_native_py315: + extends: .profiling_native_base + allow_failure: true + retry: 2 + rules: !reference [profiling_native, rules] + parallel: + matrix: + - PYTHON_VERSION: ["3.15"] + SANITIZER: ["safety", "thread", "", "valgrind"] + test-dd-sts: stage: tests needs: [] diff --git a/.gitlab/templates/build-base-venvs.yml b/.gitlab/templates/build-base-venvs.yml index dc32b3e40d8..797b73938df 100644 --- a/.gitlab/templates/build-base-venvs.yml +++ b/.gitlab/templates/build-base-venvs.yml @@ -49,3 +49,5 @@ build_base_venvs: - core.* - ddtrace/**/*.so* - .riot/venv_* + - ddtrace/internal/datadog/profiling/test/test_* + - ddtrace/internal/datadog/profiling/test/py315/test_* diff --git a/.riot/requirements/1c6cb02.txt b/.riot/requirements/1c6cb02.txt new file mode 100644 index 00000000000..b9724685738 --- /dev/null +++ b/.riot/requirements/1c6cb02.txt @@ -0,0 +1,34 @@ +# +# This file is autogenerated by pip-compile with Python 3.15 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1c6cb02.in +# +attrs==26.1.0 +coverage[toml]==7.13.5 +gunicorn==25.3.0 +hypothesis==6.45.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +numpy==2.4.4 +opentracing==2.4.0 +packaging==26.1 +pluggy==1.6.0 +protobuf==7.34.1 +py-cpuinfo==8.0.0 +pygments==2.20.0 +pytest==9.0.3 +pytest-asyncio==0.21.1 +pytest-benchmark==5.2.3 +pytest-cov==7.1.0 +pytest-cpp==2.6.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +referencing==0.37.0 +rpds-py==0.30.0 +sortedcontainers==2.4.0 +uvloop==0.22.1 +uwsgi==2.0.31 +zstandard==0.25.0 diff --git a/.riot/requirements/222bcd0.txt b/.riot/requirements/222bcd0.txt new file mode 100644 index 00000000000..4ec85bb3a9d --- /dev/null +++ b/.riot/requirements/222bcd0.txt @@ -0,0 +1,45 @@ +# +# This file is autogenerated by pip-compile with Python 3.15 +# by the following command: +# +<<<<<<<< HEAD:.riot/requirements/222bcd0.txt +# pip-compile --allow-unsafe --no-annotate .riot/requirements/222bcd0.in +# +attrs==26.1.0 +cloudpickle==3.1.2 +coverage[toml]==7.14.3 +execnet==2.1.2 +gevent==26.5.0 +greenlet==3.5.3 +httpretty==1.1.4 +======== +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1857594.in +# +attrs==26.1.0 +coverage[toml]==7.13.5 +>>>>>>>> 49c1ffeaab (ci(profiling): wire py3.15 into build matrix, riotfile, and CI):.riot/requirements/1857594.txt +hypothesis==6.45.0 +iniconfig==2.3.0 +mock==5.2.0 +opentracing==2.4.0 +packaging==26.1 +pluggy==1.6.0 +pyfakefs==6.2.0 +pygments==2.20.0 +pytest==8.4.2 +pytest-asyncio==0.23.8 +pytest-cov==7.1.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +python-json-logger==2.0.7 +sortedcontainers==2.4.0 +<<<<<<<< HEAD:.riot/requirements/222bcd0.txt +uwsgi==2.0.31 +wrapt==2.2.2 +======== +>>>>>>>> 49c1ffeaab (ci(profiling): wire py3.15 into build matrix, riotfile, and CI):.riot/requirements/1857594.txt +zope-event==5.0 +zope-interface==7.2 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==81.0.0 diff --git a/.riot/requirements/95077af.txt b/.riot/requirements/95077af.txt new file mode 100644 index 00000000000..cb9c2e03246 --- /dev/null +++ b/.riot/requirements/95077af.txt @@ -0,0 +1,33 @@ +# +# This file is autogenerated by pip-compile with Python 3.15 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/95077af.in +# +attrs==26.1.0 +coverage[toml]==7.13.5 +gunicorn==25.3.0 +hypothesis==6.45.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +numpy==2.4.4 +opentracing==2.4.0 +packaging==26.1 +pluggy==1.6.0 +protobuf==7.34.1 +py-cpuinfo==8.0.0 +pygments==2.20.0 +pytest==9.0.3 +pytest-asyncio==0.21.1 +pytest-benchmark==5.2.3 +pytest-cov==7.1.0 +pytest-cpp==2.6.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +referencing==0.37.0 +rpds-py==0.30.0 +sortedcontainers==2.4.0 +uwsgi==2.0.31 +zstandard==0.25.0 diff --git a/.riot/requirements/e26245b.txt b/.riot/requirements/e26245b.txt new file mode 100644 index 00000000000..9fc91691061 --- /dev/null +++ b/.riot/requirements/e26245b.txt @@ -0,0 +1,37 @@ +# +# This file is autogenerated by pip-compile with Python 3.15 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/e26245b.in +# +attrs==26.1.0 +coverage[toml]==7.13.5 +gevent==26.4.0 +greenlet==3.4.0 +gunicorn[gevent]==25.3.0 +hypothesis==6.45.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +numpy==2.4.4 +opentracing==2.4.0 +packaging==26.1 +pluggy==1.6.0 +protobuf==7.34.1 +py-cpuinfo==8.0.0 +pygments==2.20.0 +pytest==9.0.3 +pytest-asyncio==0.21.1 +pytest-benchmark==5.2.3 +pytest-cov==7.1.0 +pytest-cpp==2.6.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +referencing==0.37.0 +rpds-py==0.30.0 +sortedcontainers==2.4.0 +uwsgi==2.0.31 +zope-event==6.1 +zope-interface==8.3 +zstandard==0.25.0 diff --git a/riotfile.py b/riotfile.py index 10c3c9e6eaf..e783918bd63 100644 --- a/riotfile.py +++ b/riotfile.py @@ -591,7 +591,8 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT pys=select_pys(min_version="3.9", max_version="3.11"), ), Venv( - pys=select_pys(min_version="3.12"), + # TODO(py-315): 3.15 explicitly opted in for crashtracker native validation. + pys=select_pys(min_version="3.12", max_version="3.14") + ["3.15"], env={ "PYTHONWARNINGS": "ignore:This process:DeprecationWarning::", }, @@ -2197,7 +2198,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT }, ), Venv( - pys="3.14", + pys=["3.14", "3.15"], pkgs={ "grpcio": ">=1.75.0", }, @@ -3811,7 +3812,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), ], ), - # Python 3.14 - protobuf 4.22.0 is not compatible (TypeError: Metaclasses with custom tp_new) + # Python 3.14+ - protobuf 4.22.0 is not compatible (TypeError: Metaclasses with custom tp_new) Venv( pys="3.14", pkgs={"uwsgi": latest}, diff --git a/scripts/requirements_to_csv.py b/scripts/requirements_to_csv.py index ed25c87b9ef..50d041e511b 100644 --- a/scripts/requirements_to_csv.py +++ b/scripts/requirements_to_csv.py @@ -2,7 +2,7 @@ import os import re -import toml +import toml # type: ignore[import-untyped] def requirements_to_csv(): @@ -52,7 +52,7 @@ def process_deps(dependencies): if "lib-injection" in path: os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", newline="") as f: - writer = csv.writer(f) + writer = csv.writer(f, lineterminator="\n") writer.writerows(rows) diff --git a/setup.py b/setup.py index 40749a6a119..994a07fe1b8 100644 --- a/setup.py +++ b/setup.py @@ -130,6 +130,15 @@ CARGO_TARGET_DIR = NATIVE_CRATE.absolute() / f"target{sys.version_info.major}.{sys.version_info.minor}" DD_CARGO_ARGS = shlex.split(os.getenv("DD_CARGO_ARGS", "")) +# TODO(py-315): pyo3-build-config 0.27.x (max Python 3.14) may be resolved by cargo +# if the lock file is regenerated without --locked (e.g. in some CI cache scenarios). +# Setting PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 tells pyo3-build-config to bypass +# the max-version check and build via the stable ABI, which is correct since we +# already use py_limited_api="auto" in the RustExtension definition. +# pyo3 0.28+ supports Python 3.15 natively, so this is only a safety net. +if sys.version_info >= (3, 15): + os.environ.setdefault("PYO3_USE_ABI3_FORWARD_COMPATIBILITY", "1") + def _env_truthy(name: str, default: str = "0") -> bool: return os.getenv(name, default).lower() in ("1", "yes", "on", "true") @@ -1830,7 +1839,7 @@ def check_rust_toolchain(): ), ] - if sys.version_info < (3, 15): + if sys.version_info < (3, 16): _cython_sources += [ CythonExtension( "ddtrace.profiling._threading", From e7775ebe883ec818c9ed4649d7e8e1cd454c96b9 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Mon, 1 Jun 2026 16:40:17 -0400 Subject: [PATCH 06/14] refactor(profiling): replace bytecode wrapping with sys.monitoring + direct patching in _asyncio.py Co-Authored-By: Claude Sonnet 4.6 --- ddtrace/profiling/_asyncio.py | 305 ++++++++++++++++++++-------------- 1 file changed, 183 insertions(+), 122 deletions(-) diff --git a/ddtrace/profiling/_asyncio.py b/ddtrace/profiling/_asyncio.py index 9ac02e80dec..d1ca14f38e8 100644 --- a/ddtrace/profiling/_asyncio.py +++ b/ddtrace/profiling/_asyncio.py @@ -1,7 +1,6 @@ # -*- encoding: utf-8 -*- from __future__ import annotations -from functools import partial import sys from types import ModuleType import typing @@ -15,8 +14,6 @@ from ddtrace.internal.datadog.profiling import stack from ddtrace.internal.module import ModuleWatchdog from ddtrace.internal.settings.profiling import config -from ddtrace.internal.utils import get_argument_value -from ddtrace.internal.wrapping import wrap ASYNCIO_IMPORTED: bool = False @@ -70,6 +67,72 @@ def link_existing_loop_to_current_thread() -> None: _call_init_asyncio(asyncio) +# AIDEV-NOTE: sys.monitoring (Python 3.12+) is used for task-creation tracking +# on create_task / TaskGroup.create_task, where PY_RETURN gives us the new Task +# object directly. All other asyncio hooks use simple attribute replacement — +# sys.monitoring CALL/PY_START callbacks don't expose the callee's arguments, so +# there is no advantage over plain monkey-patching for those sites. +_monitoring_tool_id: typing.Optional[int] = None +# Maps id(code) -> handler(return_value) for PY_RETURN dispatch +_py_return_handlers: dict[int, typing.Callable[[typing.Any], None]] = {} + + +def _py_return_dispatch(code: typing.Any, instruction_offset: int, return_value: typing.Any) -> None: + handler = _py_return_handlers.get(id(code)) + if handler is not None: + handler(return_value) + + +def _register_return_hook(func: typing.Callable[..., typing.Any], handler: typing.Callable[[typing.Any], None]) -> bool: + """Register a sys.monitoring PY_RETURN hook for *func* on Python 3.12+. + + Returns True if the hook was installed, False if sys.monitoring is unavailable + (Python < 3.12) and the caller should fall back to monkey-patching. + """ + global _monitoring_tool_id + + if sys.version_info >= (3, 12): + m = sys.monitoring # type: ignore[attr-defined] + + if _monitoring_tool_id is None: + # Tool IDs 4-5 are free custom slots; 0-3 are reserved (debugger, coverage, + # profiler, optimizer). Try from the top to minimise conflicts. + for candidate in (5, 4): + try: + m.use_tool_id(candidate, "dd-profiling-asyncio") + m.register_callback(candidate, m.events.PY_RETURN, _py_return_dispatch) + _monitoring_tool_id = candidate + break + except ValueError: + continue + if _monitoring_tool_id is None: + return False + + try: + code = func.__code__ + _py_return_handlers[id(code)] = handler + m.set_local_events(_monitoring_tool_id, code, m.events.PY_RETURN) + return True + except Exception: + return False # nosec B110 — best-effort monitoring; fall back to monkey-patch + + return False + + +def _unregister_all_return_hooks() -> None: + """Remove all PY_RETURN monitoring hooks (called on profiler stop).""" + global _monitoring_tool_id + + _py_return_handlers.clear() + + if _monitoring_tool_id is not None and sys.version_info >= (3, 12): + try: + sys.monitoring.free_tool_id(_monitoring_tool_id) # type: ignore[attr-defined] + except Exception: # nosec B110 — best-effort cleanup + pass + _monitoring_tool_id = None + + @ModuleWatchdog.after_module_imported("asyncio") def _(asyncio: ModuleType) -> None: global ASYNCIO_IMPORTED @@ -93,168 +156,179 @@ def _get_running_loop() -> typing.Optional[aio.AbstractEventLoop]: init_stack: bool = config.stack.enabled and stack.is_available # Python 3.14+: BaseDefaultEventLoopPolicy was renamed to _BaseDefaultEventLoopPolicy - # Try both names for compatibility events_module: ModuleType = sys.modules["asyncio.events"] if sys.hexversion >= 0x030E0000: - # Python 3.14+: Use _BaseDefaultEventLoopPolicy policy_class: typing.Optional[type[typing.Any]] = getattr(events_module, "_BaseDefaultEventLoopPolicy", None) else: - # Python < 3.14: Use BaseDefaultEventLoopPolicy policy_class = getattr(events_module, "BaseDefaultEventLoopPolicy", None) if policy_class is not None: + _original_sel = policy_class.set_event_loop - @partial(wrap, policy_class.set_event_loop) # pyright: ignore[reportArgumentType] - def _( - f: typing.Callable[[object, typing.Optional[aio.AbstractEventLoop]], None], - args: typing.Any, - kwargs: typing.Any, - ) -> None: - loop: typing.Optional[aio.AbstractEventLoop] = get_argument_value(args, kwargs, 1, "loop") + def _patched_set_event_loop(self: typing.Any, loop: typing.Optional[aio.AbstractEventLoop]) -> None: if init_stack: stack.track_asyncio_loop(typing.cast(int, ddtrace_threading.current_thread().ident), loop) - return f(*args, **kwargs) + _original_sel(self, loop) + + policy_class.set_event_loop = _patched_set_event_loop if init_stack: + tasks_module: ModuleType = sys.modules["asyncio"].tasks - @partial(wrap, sys.modules["asyncio"].tasks._GatheringFuture.__init__) - def _(f: typing.Callable[..., None], args: tuple[typing.Any, ...], kwargs: dict[str, typing.Any]) -> None: - f(*args, **kwargs) - children: list[aio.Future[typing.Any]] = typing.cast( - "list[aio.Future[typing.Any]]", get_argument_value(args, kwargs, 1, "children") - ) - assert children is not None # nosec: assert is used for typing + # --- _GatheringFuture.__init__ --- + _original_gf_init = tasks_module._GatheringFuture.__init__ + def _patched_gf_init( + self: typing.Any, + children: typing.Iterable[aio.Future[typing.Any]], + *args: typing.Any, + **kwargs: typing.Any, + ) -> None: + _original_gf_init(self, children, *args, **kwargs) # TODO(py-315): current_task() raises RuntimeError on Python 3.15+ when there # is no running event loop (e.g. asyncio.gather() called outside an async # context to build a coroutine for later scheduling). In that case there is # no parent task to link from, so we skip link_tasks entirely. try: - parent: typing.Optional[aio.Task[typing.Any]] = globals()["current_task"]() + parent = globals()["current_task"]() except RuntimeError: return if parent is not None: for child in children: stack.link_tasks(parent, child) - @partial(wrap, sys.modules["asyncio"].tasks._wait) - def _( - f: typing.Callable[..., tuple[set[aio.Future[typing.Any]], set[aio.Future[typing.Any]]]], - args: tuple[typing.Any, ...], - kwargs: dict[str, typing.Any], - ) -> typing.Any: - result: tuple[set[aio.Future[typing.Any]], set[aio.Future[typing.Any]]] = f(*args, **kwargs) - futures: set[aio.Future[typing.Any]] = typing.cast( - "set[aio.Future[typing.Any]]", get_argument_value(args, kwargs, 0, "fs") - ) + tasks_module._GatheringFuture.__init__ = _patched_gf_init + + # --- asyncio.tasks._wait --- + _original_wait = tasks_module._wait + def _patched_wait(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + # fs is the first positional or the 'fs' keyword argument + fs: typing.Iterable[aio.Future[typing.Any]] = args[0] if args else kwargs.get("fs", ()) # TODO(py-315): same guard as the _GatheringFuture wrapper above — _wait may # also be invoked outside a running loop. Skip link_tasks when current_task() # raises. try: - parent: typing.Optional[aio.Task[typing.Any]] = typing.cast( - "aio.Task[typing.Any]", globals()["current_task"]() - ) + parent = typing.cast("aio.Task[typing.Any]", globals()["current_task"]()) except RuntimeError: - return result + return _original_wait(*args, **kwargs) if parent is not None: - for future in futures: + for future in fs: stack.link_tasks(parent, future) - return result - - @partial(wrap, sys.modules["asyncio"].tasks.as_completed) - def _( - f: typing.Callable[..., typing.Generator[aio.Future[typing.Any], typing.Any, None]], - args: tuple[typing.Any, ...], - kwargs: dict[str, typing.Any], - ) -> typing.Any: - loop = typing.cast("typing.Optional[aio.AbstractEventLoop]", kwargs.get("loop")) + return _original_wait(*args, **kwargs) + + tasks_module._wait = _patched_wait # type: ignore[attr-defined] + + # --- asyncio.tasks.as_completed --- + _original_as_completed = tasks_module.as_completed + + def _patched_as_completed(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + fs: typing.Iterable[aio.Future[typing.Any]] = args[0] if args else kwargs.get("fs", ()) + loop: typing.Optional[aio.AbstractEventLoop] = kwargs.get("loop") parent: typing.Optional[aio.Task[typing.Any]] = globals()["current_task"]() if parent is not None: - fs = typing.cast("typing.Iterable[aio.Future[typing.Any]]", get_argument_value(args, kwargs, 0, "fs")) futures: set[aio.Future[typing.Any]] = {asyncio.ensure_future(f, loop=loop) for f in set(fs)} for future in futures: stack.link_tasks(parent, future) - # Replace fs with the ensured futures to avoid double-wrapping. - # Handle both positional (args[0]) and keyword ('fs') call patterns: - # if fs was positional we update args; if it was a keyword we must - # update kwargs instead, otherwise f() receives fs twice and raises - # TypeError: got multiple values for argument 'fs'. if args: args = (futures,) + args[1:] else: kwargs = {**kwargs, "fs": futures} - return f(*args, **kwargs) - - # Wrap asyncio.shield to link parent task to shielded future - @partial(wrap, sys.modules["asyncio"].tasks.shield) - def _( - f: typing.Callable[..., aio.Future[typing.Any]], - args: tuple[typing.Any, ...], - kwargs: dict[str, typing.Any], - ) -> typing.Any: - loop = typing.cast("typing.Optional[aio.AbstractEventLoop]", kwargs.get("loop")) - awaitable = typing.cast("aio.Future[typing.Any]", get_argument_value(args, kwargs, 0, "arg")) + return _original_as_completed(*args, **kwargs) + + tasks_module.as_completed = _patched_as_completed # type: ignore[attr-defined] + asyncio.as_completed = _patched_as_completed # type: ignore[attr-defined] # re-export alias + + # --- asyncio.tasks.shield --- + _original_shield = tasks_module.shield + + def _patched_shield(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + loop: typing.Optional[aio.AbstractEventLoop] = kwargs.get("loop") + awaitable: aio.Future[typing.Any] = args[0] if args else kwargs["arg"] future: aio.Future[typing.Any] = asyncio.ensure_future(awaitable, loop=loop) parent: typing.Optional[aio.Task[typing.Any]] = globals()["current_task"]() if parent is not None: stack.link_tasks(parent, future) - # Same positional-vs-keyword handling as the as_completed wrapper above: - # if 'arg' was passed positionally update args, otherwise update kwargs to - # avoid TypeError: got multiple values for argument 'arg'. + # Replace the awaitable argument with the ensured future. if args: args = (future,) + args[1:] else: kwargs = {**kwargs, "arg": future} - return f(*args, **kwargs) + return _original_shield(*args, **kwargs) - # Wrap asyncio.TaskGroup.create_task to link parent task to created tasks (Python 3.11+) - if sys.hexversion >= 0x030B0000: # Python 3.11+ + tasks_module.shield = _patched_shield # type: ignore[attr-defined] + asyncio.shield = _patched_shield # type: ignore[attr-defined] # re-export alias + + # --- asyncio.TaskGroup.create_task (Python 3.11+) --- + if sys.hexversion >= 0x030B0000: taskgroups_module: typing.Optional[ModuleType] = sys.modules.get("asyncio.taskgroups") if taskgroups_module is not None: taskgroup_class: typing.Optional[type[typing.Any]] = getattr(taskgroups_module, "TaskGroup", None) if taskgroup_class is not None and hasattr(taskgroup_class, "create_task"): - @partial(wrap, taskgroup_class.create_task) - def _( - f: typing.Callable[..., aio.Task[typing.Any]], - args: tuple[typing.Any, ...], - kwargs: dict[str, typing.Any], - ) -> aio.Task[typing.Any]: - result: aio.Task[typing.Any] = f(*args, **kwargs) - - parent: typing.Optional[aio.Task[typing.Any]] = globals()["current_task"]() - if parent is not None and result is not None: - # Link parent task to the task created by TaskGroup - stack.link_tasks(parent, result) - - return result - + def _on_taskgroup_create_task_return(return_value: typing.Any) -> None: + task: typing.Optional[aio.Task[typing.Any]] = return_value + try: + parent = globals()["current_task"]() + except RuntimeError: + return + if parent is not None and task is not None: + stack.link_tasks(parent, task) + + if not _register_return_hook(taskgroup_class.create_task, _on_taskgroup_create_task_return): + # Fallback for Python 3.9-3.11: simple monkey-patch + _original_tg_create_task = taskgroup_class.create_task + + def _patched_tg_create_task( + self: typing.Any, *args: typing.Any, **kwargs: typing.Any + ) -> aio.Task[typing.Any]: + result: aio.Task[typing.Any] = _original_tg_create_task(self, *args, **kwargs) + try: + parent = globals()["current_task"]() + except RuntimeError: + return result + if parent is not None and result is not None: + stack.link_tasks(parent, result) + return result + + taskgroup_class.create_task = _patched_tg_create_task + + # --- asyncio.tasks.create_task --- # Note: asyncio.timeout and asyncio.timeout_at don't create child tasks. - # They are context managers that schedule a callback to cancel the current task - # if it times out. The timeout._task is the same as the current task, so there's - # no parent-child relationship to link. The timeout mechanism is handled by the - # event loop's timeout handler, not by creating new tasks. - @partial(wrap, sys.modules["asyncio"].tasks.create_task) - def _( - f: typing.Callable[..., aio.Task[typing.Any]], - args: tuple[typing.Any, ...], - kwargs: dict[str, typing.Any], - ) -> aio.Task[typing.Any]: - # kwargs will typically contain context (Python 3.11+ only) and eager_start (Python 3.14+ only) - task: aio.Task[typing.Any] = f(*args, **kwargs) - parent: typing.Optional[aio.Task[typing.Any]] = globals()["current_task"]() + # They are context managers that schedule a callback to cancel the current + # task if it times out; the timeout._task IS the current task, so there's + # no parent-child relationship to track. + _original_create_task = tasks_module.create_task + def _on_create_task_return(return_value: typing.Any) -> None: + task: aio.Task[typing.Any] = return_value + try: + parent = globals()["current_task"]() + except RuntimeError: + return if parent is not None: stack.weak_link_tasks(parent, task) - return task + if not _register_return_hook(_original_create_task, _on_create_task_return): + # Fallback for Python 3.9-3.11: simple monkey-patch + def _patched_create_task(*args: typing.Any, **kwargs: typing.Any) -> "aio.Task[typing.Any]": + task: "aio.Task[typing.Any]" = _original_create_task(*args, **kwargs) + try: + parent = globals()["current_task"]() + except RuntimeError: + return task + if parent is not None: + stack.weak_link_tasks(parent, task) + return task + + tasks_module.create_task = _patched_create_task # type: ignore[attr-defined] + asyncio.create_task = _patched_create_task # type: ignore[attr-defined] # re-export alias _call_init_asyncio(asyncio) @@ -270,7 +344,6 @@ def _(uvloop: ModuleType) -> None: We also hook EventLoopPolicy.set_event_loop for the deprecated uvloop.install() + asyncio.run() pattern. """ - # Check if uvloop support is disabled via configuration if not config.stack.uvloop: # pyright: ignore[reportAttributeAccessIssue] return @@ -278,46 +351,34 @@ def _(uvloop: ModuleType) -> None: init_stack: bool = config.stack.enabled and stack.is_available - # Wrap uvloop.new_event_loop to track loops when they're created new_event_loop_func: typing.Optional[typing.Callable[[], asyncio.AbstractEventLoop]] = getattr( uvloop, "new_event_loop", None ) if new_event_loop_func is not None: + _original_nel = new_event_loop_func - @partial(wrap, new_event_loop_func) # type: ignore[arg-type] - def _( - f: typing.Callable[[], asyncio.AbstractEventLoop], - args: tuple[typing.Any, ...], - kwargs: dict[str, typing.Any], - ) -> asyncio.AbstractEventLoop: - loop: asyncio.AbstractEventLoop = f(*args, **kwargs) + def _patched_new_event_loop() -> asyncio.AbstractEventLoop: + loop: asyncio.AbstractEventLoop = _original_nel() if init_stack: thread_id: int = typing.cast(int, ddtrace_threading.current_thread().ident) stack.set_uvloop_mode(thread_id, True) - stack.track_asyncio_loop(thread_id, loop) - # Ensure asyncio task tracking is initialized _call_init_asyncio(asyncio) - return loop - # Wrap uvloop.EventLoopPolicy.set_event_loop for uvloop.install() + asyncio.run() pattern + uvloop.new_event_loop = _patched_new_event_loop # type: ignore[attr-defined] + policy_class: typing.Optional[type[typing.Any]] = getattr(uvloop, "EventLoopPolicy", None) if policy_class is not None and hasattr(policy_class, "set_event_loop"): + _original_uvloop_sel = policy_class.set_event_loop - @partial(wrap, policy_class.set_event_loop) # pyright: ignore[reportArgumentType] - def _( - f: typing.Callable[[object, typing.Optional[asyncio.AbstractEventLoop]], None], - args: typing.Any, - kwargs: typing.Any, - ) -> None: - thread_id: int = typing.cast(int, ddtrace_threading.current_thread().ident) + def _patched_uvloop_set_event_loop(self: typing.Any, loop: typing.Optional[asyncio.AbstractEventLoop]) -> None: + thread_id = typing.cast(int, ddtrace_threading.current_thread().ident) if init_stack: stack.set_uvloop_mode(thread_id, True) - - loop: typing.Optional[asyncio.AbstractEventLoop] = get_argument_value(args, kwargs, 1, "loop") if init_stack and loop is not None: - stack.track_asyncio_loop(typing.cast(int, ddtrace_threading.current_thread().ident), loop) + stack.track_asyncio_loop(thread_id, loop) _call_init_asyncio(asyncio) + _original_uvloop_sel(self, loop) - return f(*args, **kwargs) + policy_class.set_event_loop = _patched_uvloop_set_event_loop From d2b6e7728e59a3bf8252e6f28b11b4c1a3fc0be2 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Thu, 4 Jun 2026 16:36:15 -0400 Subject: [PATCH 07/14] refactor(profiling): narrow sys.monitoring asyncio path to Python 3.15+ only Co-Authored-By: Claude Sonnet 4.6 --- ddtrace/profiling/_asyncio.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/ddtrace/profiling/_asyncio.py b/ddtrace/profiling/_asyncio.py index d1ca14f38e8..ff2d66ee27b 100644 --- a/ddtrace/profiling/_asyncio.py +++ b/ddtrace/profiling/_asyncio.py @@ -67,11 +67,13 @@ def link_existing_loop_to_current_thread() -> None: _call_init_asyncio(asyncio) -# AIDEV-NOTE: sys.monitoring (Python 3.12+) is used for task-creation tracking +# AIDEV-NOTE: sys.monitoring (Python 3.15+) is used for task-creation tracking # on create_task / TaskGroup.create_task, where PY_RETURN gives us the new Task # object directly. All other asyncio hooks use simple attribute replacement — # sys.monitoring CALL/PY_START callbacks don't expose the callee's arguments, so # there is no advantage over plain monkey-patching for those sites. +# TODO(py-315): Evaluate rolling sys.monitoring path out to 3.12–3.14 as a Q3 +# follow-up once there's proper CI coverage for those versions. _monitoring_tool_id: typing.Optional[int] = None # Maps id(code) -> handler(return_value) for PY_RETURN dispatch _py_return_handlers: dict[int, typing.Callable[[typing.Any], None]] = {} @@ -84,14 +86,14 @@ def _py_return_dispatch(code: typing.Any, instruction_offset: int, return_value: def _register_return_hook(func: typing.Callable[..., typing.Any], handler: typing.Callable[[typing.Any], None]) -> bool: - """Register a sys.monitoring PY_RETURN hook for *func* on Python 3.12+. + """Register a sys.monitoring PY_RETURN hook for *func* on Python 3.15+. - Returns True if the hook was installed, False if sys.monitoring is unavailable - (Python < 3.12) and the caller should fall back to monkey-patching. + Returns True if the hook was installed, False if sys.monitoring is not used + (Python < 3.15) and the caller should fall back to monkey-patching. """ global _monitoring_tool_id - if sys.version_info >= (3, 12): + if sys.version_info >= (3, 15): m = sys.monitoring # type: ignore[attr-defined] if _monitoring_tool_id is None: @@ -125,7 +127,7 @@ def _unregister_all_return_hooks() -> None: _py_return_handlers.clear() - if _monitoring_tool_id is not None and sys.version_info >= (3, 12): + if _monitoring_tool_id is not None and sys.version_info >= (3, 15): try: sys.monitoring.free_tool_id(_monitoring_tool_id) # type: ignore[attr-defined] except Exception: # nosec B110 — best-effort cleanup @@ -282,7 +284,7 @@ def _on_taskgroup_create_task_return(return_value: typing.Any) -> None: stack.link_tasks(parent, task) if not _register_return_hook(taskgroup_class.create_task, _on_taskgroup_create_task_return): - # Fallback for Python 3.9-3.11: simple monkey-patch + # Fallback for Python < 3.15: simple monkey-patch _original_tg_create_task = taskgroup_class.create_task def _patched_tg_create_task( @@ -316,7 +318,7 @@ def _on_create_task_return(return_value: typing.Any) -> None: stack.weak_link_tasks(parent, task) if not _register_return_hook(_original_create_task, _on_create_task_return): - # Fallback for Python 3.9-3.11: simple monkey-patch + # Fallback for Python < 3.15: simple monkey-patch def _patched_create_task(*args: typing.Any, **kwargs: typing.Any) -> "aio.Task[typing.Any]": task: "aio.Task[typing.Any]" = _original_create_task(*args, **kwargs) try: From a23551285ac9ff2206a8f819b04f7667e8bf503b Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Fri, 21 Aug 2026 07:52:31 +0300 Subject: [PATCH 08/14] style(profiling): rename remaining AIDEV-NOTE to TODO(py-315) Co-Authored-By: Claude Sonnet 4.6 # Conflicts: # ddtrace/profiling/collector/_memalloc_tb.cpp --- ddtrace/profiling/_asyncio.py | 2 +- ddtrace/profiling/collector/_memalloc_tb.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ddtrace/profiling/_asyncio.py b/ddtrace/profiling/_asyncio.py index ff2d66ee27b..553c75e36c9 100644 --- a/ddtrace/profiling/_asyncio.py +++ b/ddtrace/profiling/_asyncio.py @@ -67,7 +67,7 @@ def link_existing_loop_to_current_thread() -> None: _call_init_asyncio(asyncio) -# AIDEV-NOTE: sys.monitoring (Python 3.15+) is used for task-creation tracking +# TODO(py-315): sys.monitoring (Python 3.15+) is used for task-creation tracking # on create_task / TaskGroup.create_task, where PY_RETURN gives us the new Task # object directly. All other asyncio hooks use simple attribute replacement — # sys.monitoring CALL/PY_START callbacks don't expose the callee's arguments, so diff --git a/ddtrace/profiling/collector/_memalloc_tb.cpp b/ddtrace/profiling/collector/_memalloc_tb.cpp index d13c7bae381..71fe09307f8 100644 --- a/ddtrace/profiling/collector/_memalloc_tb.cpp +++ b/ddtrace/profiling/collector/_memalloc_tb.cpp @@ -141,7 +141,7 @@ traceback_t::init_sample(size_t size, size_t weighted_size, uint16_t max_nframe, push_stacktrace_to_sample_no_refcount(sample, max_nframe); } -// Constructor calls init_sample() which reads CPython structs directly +// TODO(py-315): Constructor calls init_sample() which reads CPython structs directly traceback_t::traceback_t(size_t size, size_t weighted_size, uint16_t max_nframe, PyMemAllocatorDomain domain) : sample(static_cast(Datadog::SampleType::Allocation | Datadog::SampleType::Heap), max_nframe) { From ba0a39e039816115136b35ae53755d815450b97e Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Thu, 23 Jul 2026 10:29:10 -0400 Subject: [PATCH 09/14] typing: annotate asyncio monitoring path for py3.15 --- ddtrace/profiling/_asyncio.py | 63 ++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/ddtrace/profiling/_asyncio.py b/ddtrace/profiling/_asyncio.py index 553c75e36c9..85d52f4b7b7 100644 --- a/ddtrace/profiling/_asyncio.py +++ b/ddtrace/profiling/_asyncio.py @@ -2,6 +2,7 @@ from __future__ import annotations import sys +from types import CodeType from types import ModuleType import typing @@ -76,16 +77,16 @@ def link_existing_loop_to_current_thread() -> None: # follow-up once there's proper CI coverage for those versions. _monitoring_tool_id: typing.Optional[int] = None # Maps id(code) -> handler(return_value) for PY_RETURN dispatch -_py_return_handlers: dict[int, typing.Callable[[typing.Any], None]] = {} +_py_return_handlers: dict[int, typing.Callable[[object], None]] = {} -def _py_return_dispatch(code: typing.Any, instruction_offset: int, return_value: typing.Any) -> None: - handler = _py_return_handlers.get(id(code)) +def _py_return_dispatch(code: CodeType, instruction_offset: int, return_value: object) -> None: + handler: typing.Optional[typing.Callable[[object], None]] = _py_return_handlers.get(id(code)) if handler is not None: handler(return_value) -def _register_return_hook(func: typing.Callable[..., typing.Any], handler: typing.Callable[[typing.Any], None]) -> bool: +def _register_return_hook(func: typing.Callable[..., typing.Any], handler: typing.Callable[[object], None]) -> bool: """Register a sys.monitoring PY_RETURN hook for *func* on Python 3.15+. Returns True if the hook was installed, False if sys.monitoring is not used @@ -94,11 +95,12 @@ def _register_return_hook(func: typing.Callable[..., typing.Any], handler: typin global _monitoring_tool_id if sys.version_info >= (3, 15): - m = sys.monitoring # type: ignore[attr-defined] + m: typing.Any = sys.monitoring # type: ignore[attr-defined] if _monitoring_tool_id is None: # Tool IDs 4-5 are free custom slots; 0-3 are reserved (debugger, coverage, # profiler, optimizer). Try from the top to minimise conflicts. + candidate: int for candidate in (5, 4): try: m.use_tool_id(candidate, "dd-profiling-asyncio") @@ -111,7 +113,7 @@ def _register_return_hook(func: typing.Callable[..., typing.Any], handler: typin return False try: - code = func.__code__ + code: CodeType = func.__code__ _py_return_handlers[id(code)] = handler m.set_local_events(_monitoring_tool_id, code, m.events.PY_RETURN) return True @@ -165,7 +167,7 @@ def _get_running_loop() -> typing.Optional[aio.AbstractEventLoop]: policy_class = getattr(events_module, "BaseDefaultEventLoopPolicy", None) if policy_class is not None: - _original_sel = policy_class.set_event_loop + _original_sel: typing.Callable[..., None] = policy_class.set_event_loop def _patched_set_event_loop(self: typing.Any, loop: typing.Optional[aio.AbstractEventLoop]) -> None: if init_stack: @@ -178,7 +180,7 @@ def _patched_set_event_loop(self: typing.Any, loop: typing.Optional[aio.Abstract tasks_module: ModuleType = sys.modules["asyncio"].tasks # --- _GatheringFuture.__init__ --- - _original_gf_init = tasks_module._GatheringFuture.__init__ + _original_gf_init: typing.Callable[..., None] = tasks_module._GatheringFuture.__init__ def _patched_gf_init( self: typing.Any, @@ -192,17 +194,18 @@ def _patched_gf_init( # context to build a coroutine for later scheduling). In that case there is # no parent task to link from, so we skip link_tasks entirely. try: - parent = globals()["current_task"]() + parent: typing.Optional[aio.Task[typing.Any]] = globals()["current_task"]() except RuntimeError: return if parent is not None: + child: aio.Future[typing.Any] for child in children: stack.link_tasks(parent, child) tasks_module._GatheringFuture.__init__ = _patched_gf_init # --- asyncio.tasks._wait --- - _original_wait = tasks_module._wait + _original_wait: typing.Callable[..., typing.Any] = tasks_module._wait def _patched_wait(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: # fs is the first positional or the 'fs' keyword argument @@ -211,10 +214,13 @@ def _patched_wait(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: # also be invoked outside a running loop. Skip link_tasks when current_task() # raises. try: - parent = typing.cast("aio.Task[typing.Any]", globals()["current_task"]()) + parent: typing.Optional[aio.Task[typing.Any]] = typing.cast( + "aio.Task[typing.Any]", globals()["current_task"]() + ) except RuntimeError: return _original_wait(*args, **kwargs) if parent is not None: + future: aio.Future[typing.Any] for future in fs: stack.link_tasks(parent, future) return _original_wait(*args, **kwargs) @@ -222,7 +228,7 @@ def _patched_wait(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: tasks_module._wait = _patched_wait # type: ignore[attr-defined] # --- asyncio.tasks.as_completed --- - _original_as_completed = tasks_module.as_completed + _original_as_completed: typing.Callable[..., typing.Any] = tasks_module.as_completed def _patched_as_completed(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: fs: typing.Iterable[aio.Future[typing.Any]] = args[0] if args else kwargs.get("fs", ()) @@ -231,6 +237,7 @@ def _patched_as_completed(*args: typing.Any, **kwargs: typing.Any) -> typing.Any if parent is not None: futures: set[aio.Future[typing.Any]] = {asyncio.ensure_future(f, loop=loop) for f in set(fs)} + future: aio.Future[typing.Any] for future in futures: stack.link_tasks(parent, future) # Replace fs with the ensured futures to avoid double-wrapping. @@ -245,7 +252,7 @@ def _patched_as_completed(*args: typing.Any, **kwargs: typing.Any) -> typing.Any asyncio.as_completed = _patched_as_completed # type: ignore[attr-defined] # re-export alias # --- asyncio.tasks.shield --- - _original_shield = tasks_module.shield + _original_shield: typing.Callable[..., typing.Any] = tasks_module.shield def _patched_shield(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: loop: typing.Optional[aio.AbstractEventLoop] = kwargs.get("loop") @@ -274,10 +281,12 @@ def _patched_shield(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: taskgroup_class: typing.Optional[type[typing.Any]] = getattr(taskgroups_module, "TaskGroup", None) if taskgroup_class is not None and hasattr(taskgroup_class, "create_task"): - def _on_taskgroup_create_task_return(return_value: typing.Any) -> None: - task: typing.Optional[aio.Task[typing.Any]] = return_value + def _on_taskgroup_create_task_return(return_value: object) -> None: + task: typing.Optional[aio.Task[typing.Any]] = typing.cast( + "typing.Optional[aio.Task[typing.Any]]", return_value + ) try: - parent = globals()["current_task"]() + parent: typing.Optional[aio.Task[typing.Any]] = globals()["current_task"]() except RuntimeError: return if parent is not None and task is not None: @@ -285,14 +294,16 @@ def _on_taskgroup_create_task_return(return_value: typing.Any) -> None: if not _register_return_hook(taskgroup_class.create_task, _on_taskgroup_create_task_return): # Fallback for Python < 3.15: simple monkey-patch - _original_tg_create_task = taskgroup_class.create_task + _original_tg_create_task: typing.Callable[..., aio.Task[typing.Any]] = ( + taskgroup_class.create_task + ) def _patched_tg_create_task( self: typing.Any, *args: typing.Any, **kwargs: typing.Any ) -> aio.Task[typing.Any]: result: aio.Task[typing.Any] = _original_tg_create_task(self, *args, **kwargs) try: - parent = globals()["current_task"]() + parent: typing.Optional[aio.Task[typing.Any]] = globals()["current_task"]() except RuntimeError: return result if parent is not None and result is not None: @@ -306,12 +317,12 @@ def _patched_tg_create_task( # They are context managers that schedule a callback to cancel the current # task if it times out; the timeout._task IS the current task, so there's # no parent-child relationship to track. - _original_create_task = tasks_module.create_task + _original_create_task: typing.Callable[..., aio.Task[typing.Any]] = tasks_module.create_task - def _on_create_task_return(return_value: typing.Any) -> None: - task: aio.Task[typing.Any] = return_value + def _on_create_task_return(return_value: object) -> None: + task: aio.Task[typing.Any] = typing.cast("aio.Task[typing.Any]", return_value) try: - parent = globals()["current_task"]() + parent: typing.Optional[aio.Task[typing.Any]] = globals()["current_task"]() except RuntimeError: return if parent is not None: @@ -322,7 +333,7 @@ def _on_create_task_return(return_value: typing.Any) -> None: def _patched_create_task(*args: typing.Any, **kwargs: typing.Any) -> "aio.Task[typing.Any]": task: "aio.Task[typing.Any]" = _original_create_task(*args, **kwargs) try: - parent = globals()["current_task"]() + parent: typing.Optional[aio.Task[typing.Any]] = globals()["current_task"]() except RuntimeError: return task if parent is not None: @@ -357,7 +368,7 @@ def _(uvloop: ModuleType) -> None: uvloop, "new_event_loop", None ) if new_event_loop_func is not None: - _original_nel = new_event_loop_func + _original_nel: typing.Callable[[], asyncio.AbstractEventLoop] = new_event_loop_func def _patched_new_event_loop() -> asyncio.AbstractEventLoop: loop: asyncio.AbstractEventLoop = _original_nel() @@ -372,10 +383,10 @@ def _patched_new_event_loop() -> asyncio.AbstractEventLoop: policy_class: typing.Optional[type[typing.Any]] = getattr(uvloop, "EventLoopPolicy", None) if policy_class is not None and hasattr(policy_class, "set_event_loop"): - _original_uvloop_sel = policy_class.set_event_loop + _original_uvloop_sel: typing.Callable[..., None] = policy_class.set_event_loop def _patched_uvloop_set_event_loop(self: typing.Any, loop: typing.Optional[asyncio.AbstractEventLoop]) -> None: - thread_id = typing.cast(int, ddtrace_threading.current_thread().ident) + thread_id: int = typing.cast(int, ddtrace_threading.current_thread().ident) if init_stack: stack.set_uvloop_mode(thread_id, True) if init_stack and loop is not None: From 838ff0580f3918abdd08547328c898cd23da1e22 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Thu, 23 Jul 2026 10:12:45 -0400 Subject: [PATCH 10/14] docs(profiling): add py3.15 dev tooling and CPython upgrade runbook Profiling bring-up scripts, compatibility baselines, Echion migration runbook, and py315-stack PR navigation helpers (PR 12/14). --- docs/contributing-profiling-new-cpython.rst | 474 ++++++++++++++++ docs/contributing-testing.rst | 13 + docs/contributing.rst | 31 +- docs/cpython-diffs/analysis_314_to_315.md | 227 ++++++++ scripts/profiles/compatibility_baselines.json | 101 ++++ scripts/py315-stack/DRAFT_PRS.md | 328 ++++++++++++ scripts/py315-stack/MANUAL_EXISTING_PRS.md | 42 ++ scripts/py315-stack/TRACKING.md | 79 +++ scripts/py315-stack/generate-pr-urls.sh | 179 +++++++ scripts/py315-stack/open-all-draft-prs.sh | 40 ++ scripts/py315-stack/pr-bodies/01-compare.url | 1 + .../01-gab-315-monitoring-multiplexer.md | 12 + .../02-chore-315-wrapping-context.md | 14 + scripts/py315-stack/pr-bodies/02-compare.url | 1 + scripts/py315-stack/pr-bodies/03-compare.url | 1 + .../pr-bodies/03-vlad-315-ci-matrix.md | 12 + scripts/py315-stack/pr-bodies/04-compare.url | 1 + .../04-vlad-315-ci-autoregen-lockfiles.md | 12 + scripts/py315-stack/pr-bodies/05-compare.url | 1 + .../pr-bodies/05-vlad-315-official-support.md | 12 + scripts/py315-stack/pr-bodies/06-compare.url | 1 + .../06-vlad-315-peripheral-compat.md | 12 + scripts/py315-stack/pr-bodies/07-compare.url | 1 + ...ad-profiling-native-test-install-subdir.md | 12 + scripts/py315-stack/pr-bodies/08-compare.url | 1 + .../08-vlad-ddtracepy-315-profiling-native.md | 12 + scripts/py315-stack/pr-bodies/09-compare.url | 1 + ...vlad-ddtracepy-315-profiling-collectors.md | 12 + scripts/py315-stack/pr-bodies/10-compare.url | 1 + .../10-vlad-ddtracepy-315-profiling-only.md | 12 + scripts/py315-stack/pr-bodies/11-compare.url | 1 + ...racepy-315-profiling-asyncio-monitoring.md | 12 + scripts/py315-stack/pr-bodies/12-compare.url | 1 + .../12-vlad-315-profiling-dev-tooling.md | 12 + scripts/py315-stack/pr-bodies/13-compare.url | 1 + .../13-vlad-315-lib-injection-ssi.md | 11 + scripts/py315-stack/pr-bodies/14-compare.url | 1 + .../14-vlad-315-profiling-release-note.md | 12 + scripts/py315-stack/pr-numbers.env | 15 + scripts/py315-stack/rebuild-stack.sh | 136 +++++ scripts/py315-stack/sync-pr-numbers.sh | 80 +++ scripts/run-profiling-tests | 202 +++++++ scripts/verify_profiler_compatibility.py | 506 ++++++++++++++++++ 43 files changed, 2608 insertions(+), 28 deletions(-) create mode 100644 docs/contributing-profiling-new-cpython.rst create mode 100644 docs/cpython-diffs/analysis_314_to_315.md create mode 100644 scripts/profiles/compatibility_baselines.json create mode 100644 scripts/py315-stack/DRAFT_PRS.md create mode 100644 scripts/py315-stack/MANUAL_EXISTING_PRS.md create mode 100644 scripts/py315-stack/TRACKING.md create mode 100755 scripts/py315-stack/generate-pr-urls.sh create mode 100755 scripts/py315-stack/open-all-draft-prs.sh create mode 100644 scripts/py315-stack/pr-bodies/01-compare.url create mode 100644 scripts/py315-stack/pr-bodies/01-gab-315-monitoring-multiplexer.md create mode 100644 scripts/py315-stack/pr-bodies/02-chore-315-wrapping-context.md create mode 100644 scripts/py315-stack/pr-bodies/02-compare.url create mode 100644 scripts/py315-stack/pr-bodies/03-compare.url create mode 100644 scripts/py315-stack/pr-bodies/03-vlad-315-ci-matrix.md create mode 100644 scripts/py315-stack/pr-bodies/04-compare.url create mode 100644 scripts/py315-stack/pr-bodies/04-vlad-315-ci-autoregen-lockfiles.md create mode 100644 scripts/py315-stack/pr-bodies/05-compare.url create mode 100644 scripts/py315-stack/pr-bodies/05-vlad-315-official-support.md create mode 100644 scripts/py315-stack/pr-bodies/06-compare.url create mode 100644 scripts/py315-stack/pr-bodies/06-vlad-315-peripheral-compat.md create mode 100644 scripts/py315-stack/pr-bodies/07-compare.url create mode 100644 scripts/py315-stack/pr-bodies/07-vlad-profiling-native-test-install-subdir.md create mode 100644 scripts/py315-stack/pr-bodies/08-compare.url create mode 100644 scripts/py315-stack/pr-bodies/08-vlad-ddtracepy-315-profiling-native.md create mode 100644 scripts/py315-stack/pr-bodies/09-compare.url create mode 100644 scripts/py315-stack/pr-bodies/09-vlad-ddtracepy-315-profiling-collectors.md create mode 100644 scripts/py315-stack/pr-bodies/10-compare.url create mode 100644 scripts/py315-stack/pr-bodies/10-vlad-ddtracepy-315-profiling-only.md create mode 100644 scripts/py315-stack/pr-bodies/11-compare.url create mode 100644 scripts/py315-stack/pr-bodies/11-vlad-ddtracepy-315-profiling-asyncio-monitoring.md create mode 100644 scripts/py315-stack/pr-bodies/12-compare.url create mode 100644 scripts/py315-stack/pr-bodies/12-vlad-315-profiling-dev-tooling.md create mode 100644 scripts/py315-stack/pr-bodies/13-compare.url create mode 100644 scripts/py315-stack/pr-bodies/13-vlad-315-lib-injection-ssi.md create mode 100644 scripts/py315-stack/pr-bodies/14-compare.url create mode 100644 scripts/py315-stack/pr-bodies/14-vlad-315-profiling-release-note.md create mode 100644 scripts/py315-stack/pr-numbers.env create mode 100755 scripts/py315-stack/rebuild-stack.sh create mode 100755 scripts/py315-stack/sync-pr-numbers.sh create mode 100755 scripts/run-profiling-tests create mode 100644 scripts/verify_profiler_compatibility.py diff --git a/docs/contributing-profiling-new-cpython.rst b/docs/contributing-profiling-new-cpython.rst new file mode 100644 index 00000000000..f30e44f8470 --- /dev/null +++ b/docs/contributing-profiling-new-cpython.rst @@ -0,0 +1,474 @@ +.. _profiling_new_cpython: + +Profiling and new CPython versions +================================== + +This guide is for maintainers who add support for a **new CPython minor release** (e.g. 3.15) across +**everything dd-trace-py owns in the Continuous Profiler product**: **stack** (CPU / wall samples, +Echion), **asyncio** integration for stack and tasks, **lock** profilers (threading + asyncio), +**memory** and **heap** (memalloc), **exception** profiling, **PyTorch** hook, **ddup** export, build +gates, Riot/CI, and **validation tests** for each area. + +Reference implementations: + +* `PR #15546`__ (feat(profiling): support Python 3.14) — Echion frame/task/asyncio changes, + ``setup.py`` un-gating, profiling defaults, Riot venv splits, tests, and a release note. +* `PR #17294`__ (feat(profiling): support Python 3.15) — native ABI fixes for renumbered + ``PyFrameState`` and removed ``FRAME_OWNED_BY_CSTACK``; Python-side ``_asyncio.py`` hardening + with tiered ``hasattr`` guards; new compile-time layout contract tests. + +__ https://github.com/DataDog/dd-trace-py/pull/15546 +__ https://github.com/DataDog/dd-trace-py/pull/17294 + +Current status +-------------- + +.. list-table:: + :header-rows: 1 + :widths: 15 20 65 + + * - Version + - Status + - Notes + * - 3.13 + - Supported + - Last stable release fully supported across all profiler surfaces. + * - 3.14 + - Merged (`PR #15546`__) + - Echion frame/task/asyncio, ``setup.py`` un-gating, Riot venv splits, tests done. + * - 3.15 + - `PR #17294`__ + - Echion: ``FRAME_OWNED_BY_CSTACK`` removed; ``PyFrameState`` renumbered; + ``FRAME_SUSPENDED_YIELD_FROM_LOCKED`` added for free-threaded builds. + ``_asyncio.py``: tiered ``hasattr`` guards for private asyncio APIs + (``_GatheringFuture``, ``_wait``, ``_scheduled_tasks``). + New compile-time layout contract tests (``test_cpython_layout_contracts.cpp``). + Use ``0x030f0000`` guards. + +__ https://github.com/DataDog/dd-trace-py/pull/15546 +__ https://github.com/DataDog/dd-trace-py/pull/17294 + +Version hex quick reference +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: text + + Python 3.11 → 0x030b0000 + Python 3.12 → 0x030c0000 + Python 3.13 → 0x030d0000 + Python 3.14 → 0x030e0000 ← last merged + Python 3.15 → 0x030f0000 ← current target + +Preparation: what tends to break +--------------------------------- + +When CPython bumps, expect changes in: + +* ``_PyInterpreterFrame`` and related **internal headers** (include paths move between releases; + fields may become ``_PyStackRef``, ``stackpointer`` vs ``stacktop``, ``localsplus`` layout). +* **Tagged pointers** on frame/code objects (recover ``PyObject*`` per upstream notes, e.g. + ``python/cpython#123923`` for 3.14). +* **Asyncio** C layout: ``FutureObj`` / ``TaskObj`` struct layout and where **native** tasks live + (e.g. per-thread / per-interpreter linked lists and ``asyncio_tasks_head`` in 3.14+). +* **Python-visible** asyncio: policy class renames, whether ``_scheduled_tasks`` / + ``_eager_tasks`` are exported from the C module or live only in Python. +* **Free-threaded builds** (``Py_GIL_DISABLED``): from 3.14, struct layouts diverge for nogil + builds (e.g. ``task_tid`` in ``TaskObj``). Guard with ``#ifdef Py_GIL_DISABLED`` where needed. + On Windows, ``Py_GIL_DISABLED`` must now be set explicitly by the build backend; it is no + longer inferred automatically. + +Read the PR #15546 description for the concrete 3.14 deltas before extrapolating to the next +version. + +Discover CPython deltas (before writing code) +--------------------------------------------- + +1. Use the **compare-cpython-versions skill** first — it runs a systematic diff of the headers + we depend on between two CPython tags (e.g. ``v3.14.0`` → ``v3.15.0`` or ``main``). Run it + before opening any source file: + + .. code-block:: text + + # Via the Skill tool: + compare-cpython-versions (previous: 3.14, target: 3.15) + +2. If you need to manually inspect or regenerate the diff, clone **python/cpython** (the + skill uses ``~/dd/cpython`` by convention) and diff the headers we depend on: + + .. code-block:: bash + + # Clone once (or fetch tags on an existing checkout) + git clone https://github.com/python/cpython.git ~/dd/cpython + cd ~/dd/cpython && git fetch --tags + + # Diff all headers relevant to echion/profiling between two releases + # Adjust tag names to actual release tags (e.g. v3.14.0, v3.15.0 or main) + git diff v3.14.0 v3.15.0 -- \ + Include/cpython/genobject.h \ + Include/internal/pycore_frame.h \ + Include/internal/pycore_interpframe.h \ + Include/internal/pycore_interpframe_structs.h \ + Include/internal/pycore_llist.h \ + Include/internal/pycore_runtime.h \ + Include/internal/pycore_stackref.h \ + Include/internal/pycore_tstate.h \ + Modules/_asynciomodule.c + + Key files to watch (paths can move between releases — verify they exist on the target tag): + + * ``Include/internal/pycore_interpframe_structs.h``, ``pycore_frame.h``, + ``pycore_interpframe.h``, adjacent ``pycore_*`` headers. + * ``Include/cpython/genobject.h`` and anything **PyGen_\*** / yield-from paths used in + Echion. + * ``Modules/_asynciomodule.c`` — only the struct/typedef section matters + (``FutureObj_HEAD``, ``TaskObj``, ``_Py_AsyncioModuleDebugOffsets``); function bodies + are not relevant to echion. + * ``Include/internal/pycore_tstate.h``, ``pycore_llist.h``, ``pycore_stackref.h``, + ``pycore_runtime.h`` (all became relevant in 3.14). + + A committed reference diff for 3.13 → 3.14 lives at + ``docs/cpython-diffs/cpython_313_to_314_headers.diff`` in the ``DataDog/echion`` repo. + +3. In **dd-trace-py**, use the **find-cpython-usage skill** to enumerate every internal header + and struct the codebase currently touches: + + .. code-block:: text + + # Via the Skill tool: + find-cpython-usage + +4. **Version hex:** Python 3.15 is gated with ``PY_VERSION_HEX >= 0x030f0000``. Keep older + release guards (e.g. ``0x030e0000`` for 3.14) and only add a new branch when behavior or + layout **diverges** from the prior release. + +Quick grep in dd-trace-py (find prior-version guards): + +.. code-block:: bash + + rg 'PY_VERSION_HEX|0x030e' ddtrace/internal/datadog/profiling ddtrace/profiling setup.py + rg '3, 14|3\\.14' tests ddtrace setup.py riotfile.py + +Native stack profiler (Echion) — layout in this repo +----------------------------------------------------- + +CMake extension and sources live under: + +.. code-block:: text + + ddtrace/internal/datadog/profiling/stack/ + ├── echion/echion/ # headers (frame, tasks, threads, state, greenlets, …) + │ └── cpython/tasks.h # FutureObj / TaskObj mirrors + └── src/echion/ # frame.cc, threads.cc, stack_chunk.cc, … + +(Older branches or docs may say ``stack_v2``; on current ``main`` the path is ``stack/``, defined +in ``setup.py`` as ``STACK_DIR`` under ``ddtrace/internal/datadog/profiling/stack``.) + +Typical files to revisit (mirror PR #15546): + ++---------------------------+------------------------------------------+ +| Area | Files | ++===========================+==========================================+ +| Frame ABI / includes | ``stack/echion/echion/frame.h``, | +| | ``stack/src/echion/frame.cc`` | ++---------------------------+------------------------------------------+ +| Stack chunk (frame iter) | ``stack/src/echion/stack_chunk.cc`` | ++---------------------------+------------------------------------------+ +| Task / Future layouts | ``stack/echion/echion/cpython/tasks.h`` | ++---------------------------+------------------------------------------+ +| Asyncio task enumeration | ``stack/echion/echion/tasks.h``, | +| | ``stack/echion/echion/threads.h``, | +| | ``stack/src/echion/threads.cc`` | ++---------------------------+------------------------------------------+ +| Misc guards | ``stack/echion/echion/state.h``, | +| | ``stack/echion/echion/greenlets.h`` | ++---------------------------+------------------------------------------+ + +Build against the **target** interpreter first and fix compile errors. Then run automated tests +for **stack** and **asyncio** (see `Validate all profiling features`_). + +For C/C++ conventions and safety expectations, see ``.cursor/rules/native-code.mdc`` +(if present). + +Python-side integration +----------------------- + +* ``ddtrace/profiling/_asyncio.py`` — event-loop policy names, weak sets for + scheduled/eager tasks, version-guarded access patterns. +* Search under ``ddtrace/profiling/`` for ``sys.version_info``, ``PY_MAJOR_VERSION``, and + similar. + +Build and product gating +------------------------ + +* ``setup.py`` — Ensure **memalloc**, **ddup**, and **stack** CMake extensions (and Rust + profiling features, if gated) are **not** skipped on the new Python version. PR #15546 + **removed** ``sys.version_info < (3, 14)`` style exclusions; do the same for ``(3, 15)`` + when enabling 3.15. Add a **new** upper bound only if a **future** version is known broken. + +* ``ddtrace/internal/settings/profiling.py`` — Remove any "force stack profiler off on X.Y" + guards. Keep **ddup** load failures honest: log and disable profiling when the extension + truly fails to import. + +CI, Riot, and dependencies +-------------------------- + +* ``riotfile.py`` — Add or extend ``Venv(pys="3.15", ...)`` where a new Python needs different + pins (examples from 3.14 work: **uwsgi**, **protobuf**, **gevent**, memalloc/**lz4** quirks). + Follow existing patterns for ``select_pys`` and comments explaining version caps. + +* Regenerate ``.riot/requirements/*.txt`` when adding venvs (same workflow as other Python + bumps). + +* Grep tests: ``3.14``, ``3, 14``, ``max_version``, profiling-related ``skip``. + +Wheel build images (manylinux / musllinux) +------------------------------------------ + +Linux wheels are built inside PyPA manylinux / musllinux images mirrored into +``registry.ddbuild.io`` via ``DataDog/images``. The mirrored image must contain a +``cp3XX-cp3XX`` interpreter for every Python version in the wheel matrix. When a new CPython +minor is added you have to bump the pinned image tag once upstream PyPA ships it. + +Where the image tags are referenced in this repo: + +* ``.gitlab/package.yml`` — ``MANYLINUX_AMD64_IMAGE_TAG``, ``.AARCH64_IMAGES``, + ``.X86_64_IMAGES``, plus the hard-coded ``IMAGE_TAG`` entries inside the upload-job + ``needs:`` blocks. +* ``.gitlab/benchmarks/microbenchmarks.yml`` — ``PACKAGE_IMAGE`` plus the literal + ``needs:`` job-name string that embeds the image tag. +* ``.gitlab/benchmarks/macrobenchmarks.yml`` — same ``needs:`` job-name string. +* ``.gitlab-ci.yml``, ``.gitlab/multi-os-tests.yml``, ``.gitlab/system-tests.yml``, + ``.gitlab/debugging-exploration.yml`` — additional ``IMAGE_TAG`` references for non-wheel + jobs that run inside the manylinux image. Bump these in lockstep. + +Find the right PyPA base tag +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Step 1 — list candidate Quay tags newest-first: + +.. code-block:: bash + + curl -s 'https://quay.io/api/v1/repository/pypa/manylinux2014_x86_64/tag/?limit=20&onlyActiveTags=true' \ + | python3 -c "import json,sys; [print(t['name'], t['last_modified']) for t in json.load(sys.stdin).get('tags',[])]" + +The tags follow ``YYYY.MM.DD-N``. ``latest`` always points at the newest. + +Step 2 — confirm the target ``cp3XX`` interpreter is built into the image. The authoritative +source is ``pypa/manylinux``'s ``docker/Dockerfile`` on the commit corresponding to the Quay +tag. Either ``docker run --rm quay.io/pypa/manylinux2014_x86_64: ls /opt/python`` and grep +for ``cp3XX``, or — if Docker is unavailable — read the Dockerfile directly: + +.. code-block:: bash + + # Find the commit that added the cpython version you need + gh api 'repos/pypa/manylinux/commits?path=docker/Dockerfile&per_page=30' \ + --jq '.[] | "\(.sha[0:8])\t\(.commit.author.date)\t\(.commit.message | split("\n")[0])"' \ + | grep -i "cpython 3.15" + + # Inspect the current Dockerfile to see exactly which cpython versions it builds + gh api 'repos/pypa/manylinux/contents/docker/Dockerfile' --jq '.download_url' \ + | xargs curl -sL | grep -E 'build-cpython.sh .* 3\.[0-9]+' + +Any Quay tag dated after the "add CPython 3.X" commit will carry the new interpreter. Pick the +newest one. + +Step 3 — confirm the same Quay tag exists across all four image variants we mirror: + +.. code-block:: bash + + for img in manylinux2014_x86_64 manylinux2014_aarch64 musllinux_1_2_x86_64 musllinux_1_2_aarch64; do + curl -s "https://quay.io/api/v1/repository/pypa/$img/tag/?specificTag=&onlyActiveTags=true" \ + | python3 -c "import json,sys; print('$img', 'OK' if json.load(sys.stdin).get('tags') else 'MISSING')" + done + +PyPA usually publishes all four together, but verify before relying on it. + +Mirror the tag, then bump dd-trace-py +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +1. **DataDog/images PR.** Edit ``mirror.yaml`` (search for the existing + ``quay.io/pypa/manylinux2014_x86_64`` entries) and add six new entries — one per arch for + manylinux2014 and musllinux_1_2 (x86_64, i686, aarch64) — copying the existing block's + shape and bumping the tag. Then from the repo root:: + + bzl run //image-mirroring-tooling -- update-digest quay.io/pypa/manylinux2014_x86_64: + # ...repeat for the other five sources + + Commit ``mirror.yaml`` and ``mirror.lock.yaml`` together. After merge, **wait for the + master-branch mirror job** to actually push the images to ``registry.ddbuild.io/images/mirror/pypa/…``. + +2. **Trigger the dd-trace-py internal image builds.** Mirroring alone doesn't produce the + ``v--`` tags consumed by ``.gitlab/package.yml`` (e.g. + ``v85383392-751efc0-manylinux2014_x86_64``). Manually re-run CI on ``DataDog/images`` + master for each of the four images (manylinux2014 x86_64/aarch64, + musllinux_1_2 x86_64/aarch64). Record the four new ``v...`` tags. + +3. **dd-trace-py PR.** Bump every reference listed at the top of this section to the new + tags. The ``needs:`` strings in ``microbenchmarks.yml`` / ``macrobenchmarks.yml`` embed + the literal image tag in the cross-job name; they must move in lockstep with + ``.X86_64_IMAGES`` or ``needs:`` resolution fails. + +Validate all profiling features (minor-version migration) +--------------------------------------------------------- + +Before merging support for a new CPython, treat **each profiler surface** as part of the +migration: ABI changes often break **stack** first, but **memalloc**, **locks**, and +**exceptions** use native or C API-adjacent code that must still pass on the new version. + +Automated tests (what to run) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For a quick sanity check on any Python version (import guards + pprof samples), use the +compatibility script before running the full suite: + +.. code-block:: bash + + # Import/guard checks only — no C extensions required (~2 s) + python scripts/verify_profiler_compatibility.py --python 3.15 --quick + + # Full check: asyncio guards + real pprof samples with named tasks (~8 s) + python scripts/verify_profiler_compatibility.py --python 3.15 + + # Save results as the baseline for this MAJOR.MINOR + python scripts/verify_profiler_compatibility.py --python 3.15 --baseline + + # Compare against a saved baseline (use in CI or after a change) + python scripts/verify_profiler_compatibility.py --python 3.15 --compare + +Baselines for Python 3.9–3.14 live in ``scripts/profiles/compatibility_baselines.json``. + +Use **`scripts/run-tests`** (see :ref:`testing_guidelines` in ``contributing-testing``) — +**never** raw ``pytest`` for full-suite validation. For profiling, CI maps paths to Riot via +**`tests/profiling/suitespec.yml`**: patterns such as **`profile$`**, **`profile-uwsgi`**, and +**`profile-memalloc`**. + +**Feature → code → tests** (paths relative to ``ddtrace/profiling/`` or +``tests/profiling/``): + +* **Stack / wall / CPU** — ``collector/stack.py`` and Echion under + ``ddtrace/internal/datadog/profiling/stack/``. Tests: ``collector/test_stack.py``, + ``collector/test_stack_native.py``, ``test_accuracy.py``, and the many + ``collector/test_asyncio_*.py`` files for asyncio stack semantics. + +* **Locks** — ``collector/threading.py``, ``collector/asyncio.py``, ``collector/_lock.pyx``. + Tests: ``collector/test_threading.py``, ``collector/test_lock_reflection.py``, + ``collector/lock_test_common.py``, plus asyncio tests that cover lock collectors. + +* **Memory (allocations)** — ``collector/memalloc.py`` and ``collector/_memalloc*``. Tests: + ``collector/test_memalloc.py``, ``test_memalloc_fork.py``, + ``collector/test_copy_memory_stats.py``. + +* **Heap (live)** — same memalloc pipeline; ``collector/test_heap_tracker_count.py``. + +* **Exceptions** — ``collector/exception.py``; ``collector/test_exception.py``. + +* **PyTorch** — ``collector/pytorch.py``; ``test_pytorch.py``. + +* **Profiler / scheduler** — ``profiler.py``, ``scheduler.py``; ``test_profiler.py``, + ``test_scheduler.py``, ``test_profiling_config.py``. + +* **ddup / export** — internal ddup + ``tests/profiling/exporter/test_ddup.py``. + +**Practical matrix:** + +* **Stack / Echion / asyncio framing:** run the **profile** suite (``profile$``); include + ``collector/test_stack_native.py`` and representative ``test_asyncio_*.py`` files while + iterating. +* **Memalloc / heap:** run **profile-memalloc**; always include ``collector/test_memalloc.py`` + and ``collector/test_heap_tracker_count.py``. +* **Locks / threading:** use ``collector/test_threading.py`` and related asyncio lock tests + (file is large — narrow with ``run-tests`` on touched paths during development, then full + profile suite before merge). +* **Full profiling regression:** ``scripts/run-tests`` over ``tests/profiling/`` or let the + script pick venvs from changed files; locally mirror CI with ``riot run …`` **profile$** / + **profile-memalloc** / **profile-uwsgi** as needed. + +**New code paths** (new env flag, CPython branch, or collector behavior) should get **unit or +subprocess tests** next to the nearest file above; follow existing patterns (many tests use +``@pytest.mark.subprocess`` and init helpers in ``tests/profiling/collector/conftest.py``). + +Manual / dogfood checks (optional but recommended) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Automation does not replace **real workloads** or **Profiling Explorer** behavior. On staging +or a one-off service, with **Python version + ddtrace commit + ``DD_PROFILING_*``** documented: + +* **Stack / CPU:** visible stacks and CPU/wall samples; timeline if enabled. +* **Locks:** lock / lock-wait views; exercise ``threading`` and ``asyncio`` primitives; if + using **``DD_PROFILING_LOCK_EXCLUDE_MODULES``**, compare with it unset vs set. +* **Memory / heap:** allocation and live-heap signal under load. +* **Exceptions:** exception profiling after controlled errors. +* **PyTorch:** small torch workload when that collector is enabled. +* **Export:** optional **``DD_PROFILING_OUTPUT_PPROF``** for local pprof inspection. + +Staging service experiment (recommended for minor CPython bumps) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For migration confidence, mirror how you might run a **targeted staging rollout** (e.g. a +Cython- or profiler-related change on an internal worker/API): one **representative service**, +fixed traffic or time window, **documented** build and env. + +**Goal:** Prove the new interpreter + dd-trace-py candidate do not regress **runtime health** +or **profiler signal** under real workloads — not only that unit tests pass. + +**Pick a service** that stresses what you changed and what we own: + +* **Stack / Echion:** mixed CPU work, deep stacks, **asyncio** (native tasks if you touched + task enumeration), optional **gevent**/event-loop variants if the app uses them. +* **Locks:** workloads with ``threading`` and ``asyncio`` sync primitives (same idea as lock- + profiler staging). +* **Memory / heap:** allocations + longer-lived objects if memalloc paths changed. +* **Exceptions:** paths that raise and catch often enough to see exception profiles. + +**Experiment design (minimal):** + +#. **Baseline arm:** current production-like combo (CPython + ddtrace version) on staging, + same **service** and **approximate load** (QPS, soak duration). +#. **Candidate arm:** **only** CPython and/or ddtrace bump (e.g. wheel from your PR build); + keep other deps, feature flags, and ``DD_*`` **as equal as possible**. +#. Record **commit SHAs**, **artifact** (wheel/sdist), **Python ``sys.version``**, and all + relevant **`DD_PROFILING_*`**, **`DD_TRACE_*`**, and injection settings for both arms. + +**What to watch (staging / Observability):** + +* **Health:** error rate, latency, CPU/memory, restarts/crashes, OOMs. +* **Profiler product:** absence of **profiler** client errors/logs; expected **profile types** + still arriving (CPU/wall, allocation, lock/lock-wait, exceptions, heap if enabled). +* **Profiling Explorer:** open the **same** service + environment + time range pattern for each + arm; spot-check **flame graphs**, **lock** facets, **allocations**, **exceptions** for + sensible stacks and no obvious holes after the version bump. + +**Optional A/B on profiler knobs:** If validating a profiler-only change (e.g. lock exclude +list), run **two** candidate configs — **full wrap** vs **service-tuned excludes** — with +identical CPython and ddtrace versions so overhead/signal tradeoffs are isolated. + +**Duration and rollback:** Prefer at least one **full business-day** soak or replayed load; +define **rollback** (revert image or pin) if crash rate, SLO breach, or missing profiles +exceed agreed thresholds. + +**Handoff:** Paste the arm summary (versions, env, links to Explorer time ranges) into the PR +or JIRA so reviewers can reproduce the staging story. + +Release notes +~~~~~~~~~~~~~ + +* Add a **release note** with the **releasenote** skill (``AGENTS.md``). +* Smoke / telemetry / serverless: grep for version conditionals if profiling availability + changed (see files touched in PR #15546). + +Suggested order of work +------------------------ + +#. CPython header/asyncio diff + in-repo grep for the previous release's ``PY_VERSION_HEX`` / + version tuples (use the **compare-cpython-versions** and **find-cpython-usage** skills). +#. Echion (vendor copy): compile on target Python; fix ``#if`` ladders and struct/layout + drift header by header. +#. Apply asyncio task struct / linked-list changes (``cpython/tasks.h``, ``threads.cc``). +#. ``_asyncio.py`` and any other Python-side version branches. +#. ``setup.py`` and ``ddtrace/internal/settings/profiling.py`` gating. +#. Riot, requirements files, test skip cleanup. +#. **Validate all profiling features** with automated tests (matrix above) on the target + Python. +#. **Staging service experiment** (above) for at least one representative workload, or + narrower manual / dogfood checks if staging access is limited. +#. Release note and final CI green. diff --git a/docs/contributing-testing.rst b/docs/contributing-testing.rst index 3650cc16ed1..15ab4dfc52b 100644 --- a/docs/contributing-testing.rst +++ b/docs/contributing-testing.rst @@ -53,6 +53,19 @@ The ``scripts/run-tests`` script handles this automatically: $ scripts/run-tests tests/contrib/django/ $ scripts/run-tests tests/contrib/flask/test_flask.py +**Profiling on a new Python version** + +When bringing up Continuous Profiler support for a new CPython (for example 3.15), +use ``scripts/run-profiling-tests`` to run the compatibility smoke script and riot +profiling suites in one pass: + +.. code-block:: bash + + $ scripts/run-profiling-tests --python 3.15 + +See also :doc:`contributing-profiling-new-cpython` for the native stack-profiler +migration checklist. + **Manual approach with ddtest** This repo includes a Docker container definition that provides a pre-built test environment. diff --git a/docs/contributing.rst b/docs/contributing.rst index 8e1be57936f..4e7b8e85496 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -18,6 +18,8 @@ If you're trying to set up a local development environment, read `this `_. +`Profiling and new CPython versions `_ (stack profiler / Echion migration checklist). + Thanks for working with us! .. _change_process: @@ -192,34 +194,6 @@ call is ``add_integration``, which generates telemetry data about the integratio Read the docstrings in ``ddtrace/internal/telemetry/writer.py`` for more comprehensive usage information about Instrumentation Telemetry. -Configuration Registry ----------------------- - -When you add a new ``DD_*`` or ``OTEL_*`` environment variable to ddtrace, you must also register it in -two places. - -**1. Local registry** (``supported-configurations.json``) - -Add an entry for the new variable following the existing schema, then regenerate the generated module:: - - python scripts/supported_configurations.py - -Stage and commit both ``supported-configurations.json`` and -``ddtrace/internal/settings/_supported_configurations.py``. The pre-commit hook will catch any mismatch -before commit, and the CI ``check`` job will enforce it on every PR. - -**2. Central Configuration Registry** (FPD) - -The key must also be added to the `Datadog Configuration Registry `_ -by an **internal contributor**. If the key already exists with the same settings (default value, type) -because another tracer language already registered it, this step can be skipped. If the existing entry's -data doesn't match (e.g. different type or default), create a new implementation version in the registry -and reference that version's letter in ``supported-configurations.json``. - -Not adding the config and implementation details to the central registry will cause the -``validate_supported_configurations_v2_local_file`` GitLab CI job to fail, displaying the missing keys -in its output. A Datadog maintainer must add the key to the registry before the PR can merge. - .. toctree:: :hidden: @@ -227,6 +201,7 @@ in its output. A Datadog maintainer must add the key to the registry before the contributing-integrations contributing-testing contributing-fuzzing + contributing-profiling-new-cpython contributing-tracing contributing-release releasenotes diff --git a/docs/cpython-diffs/analysis_314_to_315.md b/docs/cpython-diffs/analysis_314_to_315.md new file mode 100644 index 00000000000..8019cb89a69 --- /dev/null +++ b/docs/cpython-diffs/analysis_314_to_315.md @@ -0,0 +1,227 @@ +# CPython 3.14 → 3.15 Change Analysis (for echion) + +**Generated from:** `git diff v3.14.0 v3.15.0a7` on `python/cpython` +**Latest 3.15 tag used:** `v3.15.0a7` (pre-release; verify against final tag when available) +**Raw diff:** `cpython_314_to_315_headers.diff` (1,479 lines) + +Files with **no changes** relevant to echion (stable between 3.14 and 3.15): + +- `Include/cpython/genobject.h` — generator object layout unchanged +- `Include/internal/pycore_llist.h` — llist API unchanged +- `Include/internal/pycore_runtime.h` — runtime struct unchanged + +--- + +## Breaking Changes (must fix to compile/run correctly) + +### 1. `PyFrameState` enum completely renumbered — `pycore_frame.h` + +**Priority: HIGH** + +| State | 3.14 value | 3.15 value | +|---|---|---| +| `FRAME_CREATED` | -3 | 0 | +| `FRAME_SUSPENDED` | -2 | 1 | +| `FRAME_SUSPENDED_YIELD_FROM` | -1 | 2 | +| `FRAME_SUSPENDED_YIELD_FROM_LOCKED` | *(new)* | 3 | +| `FRAME_EXECUTING` | 0 | 4 | +| `FRAME_COMPLETED` | 1 | *(removed)* | +| `FRAME_CLEARED` | 4 | 5 | + +Changed macros: + +```c +// 3.14 +#define FRAME_STATE_SUSPENDED(S) ((S) == FRAME_SUSPENDED || (S) == FRAME_SUSPENDED_YIELD_FROM) +#define FRAME_STATE_FINISHED(S) ((S) >= FRAME_COMPLETED) + +// 3.15 +#define FRAME_STATE_SUSPENDED(S) ((S) >= FRAME_SUSPENDED && (S) <= FRAME_SUSPENDED_YIELD_FROM_LOCKED) +#define FRAME_STATE_FINISHED(S) ((S) == FRAME_CLEARED) +``` + +**Echion impact:** +- Any code reading `_PyInterpreterFrame.f_frame_state` and comparing against old + constants will silently misclassify frames (e.g., `FRAME_EXECUTING = 0` in 3.14 + now means `FRAME_CREATED` in 3.15). +- `FRAME_COMPLETED` is gone — code checking `>= FRAME_COMPLETED` will break. +- New `FRAME_SUSPENDED_YIELD_FROM_LOCKED` needs to be included in suspended checks. +- **Use the `FRAME_STATE_SUSPENDED` / `FRAME_STATE_FINISHED` macros** instead of + hardcoding values, so the `#if PY_VERSION_HEX` guard only needs to cover the + macro definitions, not every use site. + +**Files to update:** `echion/frame.h`, `echion/state.h`, any caller that checks +`frame_state` directly. + +**Guard:** `#if PY_VERSION_HEX >= 0x030f0000` + +--- + +### 2. `FRAME_OWNED_BY_CSTACK` removed — `pycore_interpframe_structs.h` + +**Priority: LOW** + +```c +// 3.14 +enum _frameowner { + FRAME_OWNED_BY_THREAD = 0, + FRAME_OWNED_BY_GENERATOR = 1, + FRAME_OWNED_BY_FRAME_OBJECT = 2, + FRAME_OWNED_BY_INTERPRETER = 3, + FRAME_OWNED_BY_CSTACK = 4, // <-- removed in 3.15 +}; +``` + +**Echion impact:** If any code checks `frame->owner == FRAME_OWNED_BY_CSTACK`, +wrap in `#if PY_VERSION_HEX < 0x030f0000`. + +--- + +### 3. `_PyStackRef` tag scheme unified — `pycore_stackref.h` + +**Priority: MEDIUM** (mostly affects free-threaded builds) + +Key changes: + +- Tag constants moved to top-level (no longer split between GIL/nogil paths): + ```c + #define Py_INT_TAG 3 + #define Py_TAG_INVALID 2 // new: marks ERROR sentinel + #define Py_TAG_REFCNT 1 + #define Py_TAG_BITS 3 + #define Py_TAGGED_SHIFT 2 // new + ``` +- `Py_TAG_DEFERRED` (free-threaded) is **gone** — merged with `Py_TAG_REFCNT`. +- `PyStackRef_FromPyObjectImmortal()` **renamed** to `PyStackRef_FromPyObjectBorrow()`. +- New `PyStackRef_ERROR` sentinel (`bits == Py_TAG_INVALID`). +- New predicates: `PyStackRef_IsError()`, `PyStackRef_IsMalformed()`, + `PyStackRef_IsValid()`. +- New `PyStackRef_Wrap()` / `PyStackRef_Unwrap()` for raw pointer wrapping. +- `INITIAL_STACKREF_INDEX` changed from `8` to `(5 << Py_TAGGED_SHIFT)` = `20`. +- Tagged int shift changed: `(i << 2)` instead of `(i << 2)` — same for non-debug, + but `Py_TAGGED_SHIFT = 2` is now the canonical name. + +**Echion impact:** +- The `PyStackRef_AsPyObjectBorrow(f->f_executable)` call to recover a `PyObject*` + from a frame's executable field **still works** — no change to the public API. +- If echion directly manipulates `.bits` (e.g., checking `(bits & 1)`), update to + use the new named constants. +- If echion uses `PyStackRef_FromPyObjectImmortal()`, rename to + `PyStackRef_FromPyObjectBorrow()` under a `#if PY_VERSION_HEX >= 0x030f0000` guard. +- Free-threaded builds: `Py_TAG_DEFERRED` no longer exists; use `Py_TAG_REFCNT`. + +--- + +## Additive / Beneficial Changes (no breakage, consider adopting) + +### 4. `_PyFrame_SafeGetCode()` and `_PyFrame_SafeGetLasti()` — `pycore_interpframe.h` + +New in 3.15, **explicitly designed for profilers and debuggers**: + +```c +// Returns NULL if frame is invalid or freed (heuristic, not 100% reliable) +static inline PyCodeObject* _Py_NO_SANITIZE_THREAD +_PyFrame_SafeGetCode(_PyInterpreterFrame *f); + +// Returns -1 if frame is invalid or freed +static inline int _Py_NO_SANITIZE_THREAD +_PyFrame_SafeGetLasti(struct _PyInterpreterFrame *f); +``` + +**Recommendation:** Under `#if PY_VERSION_HEX >= 0x030f0000`, use +`_PyFrame_SafeGetCode()` instead of `_PyFrame_GetCode()` in echion's frame-reading +path. It checks for freed memory (globals/builtins NULL, `_PyMem_IsPtrFreed`, +`_PyObject_IsFreed`, `PyCode_Check`) before dereferencing. + +--- + +### 5. `base_frame` sentinel in `_PyThreadStateImpl` — `pycore_tstate.h` + +New field, **specifically called out as for profiling/sampling**: + +```c +typedef struct _PyThreadStateImpl { + PyThreadState base; + + // Embedded base frame - sentinel at the bottom of the frame stack. + // Used by profiling/sampling to detect incomplete stack traces. + _PyInterpreterFrame base_frame; // <-- NEW in 3.15 + + // ... + Py_ssize_t refcount; +``` + +**Recommendation:** Use `&tstate_impl->base_frame` as the termination sentinel when +walking the frame chain under 3.15. Previously echion checked for NULL +`previous_instr` or similar; this explicit sentinel is cleaner. + +Guard: `#if PY_VERSION_HEX >= 0x030f0000` + +--- + +### 6. `_Py_AsyncioDebug` symbol rename — `_asynciomodule.c` + +```c +// 3.14 +GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets _AsyncioDebug) + +// 3.15 +GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets _Py_AsyncioDebug) +``` + +**Echion impact:** Only relevant if echion reads this debug symbol by name from the +process (e.g., via `/proc/pid/maps` or DWARF). Update the symbol name lookup to +`_Py_AsyncioDebug` under a `#if PY_VERSION_HEX >= 0x030f0000` guard. + +The `TaskObj` struct layout (fields: `task_name`, `task_awaited_by`, `task_coro`, +`task_node`, `task_is_task`, `task_awaited_by_is_set`) is **unchanged** from 3.14 — +the `cpython/tasks.h` mirror in echion does not need layout changes. + +--- + +### 7. Other `_PyThreadStateImpl` additions — `pycore_tstate.h` + +New fields (low echion impact): + +- `c_stack_init_base` / `c_stack_init_top` — stack protection reset values +- `generator_return_kind` enum — distinguishes yield vs return in `gen_send_ex2()` +- `pystats_struct` (under `Py_STATS`) +- `jit_tracer_state` (under `_Py_TIER2`) +- `__padding[64]` (GIL-disabled, cache-line alignment) + +These add fields **after** `asyncio_running_loop` / `asyncio_tasks_head`, so if +echion accesses those by name (not by offset), no change needed. If accessing by +raw offset, regenerate offsets. + +--- + +## Work checklist for echion 3.15 port + +- [x] Add `#if PY_VERSION_HEX >= 0x030f0000` guard with new `PyFrameState` values + (renumbered) and new `FRAME_SUSPENDED_YIELD_FROM_LOCKED` state. + → `tasks.h`: new `PyGen_yf` branch for 3.15; `FRAME_SUSPENDED_YIELD_FROM_LOCKED` + is only reachable in free-threaded builds so it is guarded with + `#ifdef Py_GIL_DISABLED`. GIL builds behave identically to 3.14. +- [x] Update `FRAME_STATE_SUSPENDED` / `FRAME_STATE_FINISHED` usage to use macros. + → Not applicable: echion uses enum constants by name (not hardcoded values), + so the renumbering has no effect. `FRAME_STATE_SUSPENDED`/`FRAME_STATE_FINISHED` + macros are not used in echion code. +- [x] Remove any reference to `FRAME_COMPLETED` under 3.15 path. + → Not applicable: `FRAME_COMPLETED` is not referenced in echion's codebase. +- [x] Remove `FRAME_OWNED_BY_CSTACK` reference under 3.15 guard. + → `frame.cc`: split `>= 0x030e0000` into `>= 0x030f0000` (no CSTACK) and + `>= 0x030e0000` (CSTACK + INTERPRETER). Also fixed `is_entry` assignment. +- [x] Rename `PyStackRef_FromPyObjectImmortal` → `PyStackRef_FromPyObjectBorrow` + (if used) under 3.15 guard. + → Not applicable: `PyStackRef_FromPyObjectImmortal` is not used in echion's codebase. +- [ ] Consider adopting `_PyFrame_SafeGetCode()` for safer frame reading. +- [ ] Consider using `base_frame` sentinel for frame-chain termination. +- [x] Update asyncio debug symbol lookup: `_AsyncioDebug` → `_Py_AsyncioDebug`. + → Not applicable: echion does not look up the asyncio debug symbol by name. +- [x] Update CI/build matrix: add `cp315-*` wheels, Python 3.15 test variants. + → Done: `pyproject.toml`, `riotfile.py`, `.gitlab/package.yml`, + `.gitlab/testrunner.yml`, `.gitlab/templates/build-base-venvs.yml`, + `.gitlab/templates/detect-global-locks.yml`, `.gitlab/multi-os-tests.yml`, + `.gitlab-ci.yml`, `.github/workflows/generate-package-versions.yml`, + `.github/workflows/generate-supported-versions.yml`. +- [ ] Run echion test suite against a CPython 3.15 build and confirm green. diff --git a/scripts/profiles/compatibility_baselines.json b/scripts/profiles/compatibility_baselines.json new file mode 100644 index 00000000000..2776f9819e0 --- /dev/null +++ b/scripts/profiles/compatibility_baselines.json @@ -0,0 +1,101 @@ +{ + "_comment": "Baselines generated by scripts/verify_profiler_compatibility.py --baseline. Update by running on a known-good version.", + "3.9": { + "asyncio_guards": { + "passed": true + }, + "profiler_samples": { + "passed": true, + "min_wall_time_samples": 5, + "asyncio_task_names_seen": [ + "compat-task-0", + "compat-task-1", + "compat-task-2" + ] + } + }, + "3.10": { + "asyncio_guards": { + "passed": true + }, + "profiler_samples": { + "passed": true, + "min_wall_time_samples": 5, + "asyncio_task_names_seen": [ + "compat-task-0", + "compat-task-1", + "compat-task-2" + ] + } + }, + "3.11": { + "asyncio_guards": { + "passed": true + }, + "profiler_samples": { + "passed": true, + "min_wall_time_samples": 5, + "asyncio_task_names_seen": [ + "compat-task-0", + "compat-task-1", + "compat-task-2" + ] + } + }, + "3.12": { + "asyncio_guards": { + "passed": true + }, + "profiler_samples": { + "passed": true, + "min_wall_time_samples": 5, + "asyncio_task_names_seen": [ + "compat-task-0", + "compat-task-1", + "compat-task-2" + ] + } + }, + "3.13": { + "asyncio_guards": { + "passed": true + }, + "profiler_samples": { + "passed": true, + "min_wall_time_samples": 5, + "asyncio_task_names_seen": [ + "compat-task-0", + "compat-task-1", + "compat-task-2" + ] + } + }, + "3.14": { + "asyncio_guards": { + "passed": true + }, + "profiler_samples": { + "passed": true, + "min_wall_time_samples": 5, + "asyncio_task_names_seen": [ + "compat-task-0", + "compat-task-1", + "compat-task-2" + ] + } + }, + "3.15": { + "asyncio_guards": { + "passed": true + }, + "profiler_samples": { + "passed": true, + "min_wall_time_samples": 2, + "asyncio_task_names_seen": [ + "compat-task-0", + "compat-task-1", + "compat-task-2" + ] + } + } +} diff --git a/scripts/py315-stack/DRAFT_PRS.md b/scripts/py315-stack/DRAFT_PRS.md new file mode 100644 index 00000000000..d98ff4e143d --- /dev/null +++ b/scripts/py315-stack/DRAFT_PRS.md @@ -0,0 +1,328 @@ +# Python 3.15 stack — draft PRs (1–14) + +EMU blocks `gh pr create` / `gh pr edit`. Run [`sync-pr-numbers.sh`](sync-pr-numbers.sh) then [`open-all-draft-prs.sh`](open-all-draft-prs.sh). +Set numbers: `./sync-pr-numbers.sh PR1=19250 PR6=19255 ...` (preserves existing, fills gaps from GitHub). + +## 1/14 — `gab/315-monitoring-multiplexer` → `main` + +**Existing PR:** [#19247](https://github.com/DataDog/dd-trace-py/pull/19247) — set base to `main`, mark draft, paste body from [`pr-bodies/01-gab-315-monitoring-multiplexer.md`](pr-bodies/01-gab-315-monitoring-multiplexer.md). + +**prev:** — | **next:** [#17849](https://github.com/DataDog/dd-trace-py/pull/17849) + +
PR body (copy) + +**prev:** — | **next:** [#17849](https://github.com/DataDog/dd-trace-py/pull/17849) + +## Summary + +Introduces `ddtrace.internal.monitoring` — shared `sys.monitoring` multiplexer (Gab). Split from #17849. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + + +
+ +## 2/14 — `chore/315-wrapping-context` → `gab/315-monitoring-multiplexer` + +**Existing PR:** [#17849](https://github.com/DataDog/dd-trace-py/pull/17849) — set base to `gab/315-monitoring-multiplexer`, mark draft, paste body from [`pr-bodies/02-chore-315-wrapping-context.md`](pr-bodies/02-chore-315-wrapping-context.md). + +**prev:** [#19247](https://github.com/DataDog/dd-trace-py/pull/19247) | **next:** [#19253](https://github.com/DataDog/dd-trace-py/pull/19253) + +
PR body (copy) + +**prev:** [#19247](https://github.com/DataDog/dd-trace-py/pull/19247) | **next:** [#19253](https://github.com/DataDog/dd-trace-py/pull/19253) + +## Summary + +Wrapping context + bytecode_injection for 3.15 (Gab + await/send fix). + + +Retarget existing **#17849** to base `gab/315-monitoring-multiplexer`. + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + + +
+ +## 3/14 — `vlad/315-ci-matrix` → `gab/315-wrapping-context` + +**Existing PR:** [#19253](https://github.com/DataDog/dd-trace-py/pull/19253) — set base to `gab/315-wrapping-context`, mark draft, paste body from [`pr-bodies/03-vlad-315-ci-matrix.md`](pr-bodies/03-vlad-315-ci-matrix.md). + +**prev:** [#17849](https://github.com/DataDog/dd-trace-py/pull/17849) | **next:** [#19252](https://github.com/DataDog/dd-trace-py/pull/19252) + +
PR body (copy) + +**prev:** [#17849](https://github.com/DataDog/dd-trace-py/pull/17849) | **next:** [#19252](https://github.com/DataDog/dd-trace-py/pull/19252) + +## Summary + +Riotfile 3.15, docker testrunner, suite gating. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + + +
+ +## 4/14 — `vlad/315-ci-autoregen-lockfiles` → `vlad/315-ci-matrix` + +**Existing PR:** [#19252](https://github.com/DataDog/dd-trace-py/pull/19252) — set base to `vlad/315-ci-matrix`, mark draft, paste body from [`pr-bodies/04-vlad-315-ci-autoregen-lockfiles.md`](pr-bodies/04-vlad-315-ci-autoregen-lockfiles.md). + +**prev:** [#19253](https://github.com/DataDog/dd-trace-py/pull/19253) | **next:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) + +
PR body (copy) + +**prev:** [#19253](https://github.com/DataDog/dd-trace-py/pull/19253) | **next:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) + +## Summary + +Self-healing lockfile drift on PR branches. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + + +
+ +## 5/14 — `vlad/315-official-support` → `vlad/315-ci-autoregen-lockfiles` + +**Existing PR:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) — set base to `vlad/315-ci-autoregen-lockfiles`, mark draft, paste body from [`pr-bodies/05-vlad-315-official-support.md`](pr-bodies/05-vlad-315-official-support.md). + +**prev:** [#19252](https://github.com/DataDog/dd-trace-py/pull/19252) | **next:** [#19255](https://github.com/DataDog/dd-trace-py/pull/19255) + +
PR body (copy) + +**prev:** [#19252](https://github.com/DataDog/dd-trace-py/pull/19252) | **next:** [#19255](https://github.com/DataDog/dd-trace-py/pull/19255) + +## Summary + +pyproject.toml, requirements.csv, riot lockfiles. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + + +
+ +## 6/14 — `vlad/315-peripheral-compat` → `vlad/315-official-support` + +**Existing PR:** [#19255](https://github.com/DataDog/dd-trace-py/pull/19255) — set base to `vlad/315-official-support`, mark draft, paste body from [`pr-bodies/06-vlad-315-peripheral-compat.md`](pr-bodies/06-vlad-315-peripheral-compat.md). + +**prev:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) | **next:** [#19257](https://github.com/DataDog/dd-trace-py/pull/19257) + +
PR body (copy) + +**prev:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) | **next:** [#19257](https://github.com/DataDog/dd-trace-py/pull/19257) + +## Summary + +Graceful degradation + test skips outside wrapping core. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + + +
+ +## 7/14 — `vlad/profiling-native-test-install-subdir` → `vlad/315-peripheral-compat` + +**Existing PR:** [#19257](https://github.com/DataDog/dd-trace-py/pull/19257) — set base to `vlad/315-peripheral-compat`, mark draft, paste body from [`pr-bodies/07-vlad-profiling-native-test-install-subdir.md`](pr-bodies/07-vlad-profiling-native-test-install-subdir.md). + +**prev:** [#19255](https://github.com/DataDog/dd-trace-py/pull/19255) | **next:** [#19250](https://github.com/DataDog/dd-trace-py/pull/19250) + +
PR body (copy) + +**prev:** [#19255](https://github.com/DataDog/dd-trace-py/pull/19255) | **next:** [#19250](https://github.com/DataDog/dd-trace-py/pull/19250) + +## Summary + +INSTALL_SUBDIR for py3.15 native tests. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + + +
+ +## 8/14 — `vlad/ddtracepy-315-profiling-native` → `vlad/profiling-native-test-install-subdir` + +**Existing PR:** [#19250](https://github.com/DataDog/dd-trace-py/pull/19250) — set base to `vlad/profiling-native-test-install-subdir`, mark draft, paste body from [`pr-bodies/08-vlad-ddtracepy-315-profiling-native.md`](pr-bodies/08-vlad-ddtracepy-315-profiling-native.md). + +**prev:** [#19257](https://github.com/DataDog/dd-trace-py/pull/19257) | **next:** [#19251](https://github.com/DataDog/dd-trace-py/pull/19251) + +
PR body (copy) + +**prev:** [#19257](https://github.com/DataDog/dd-trace-py/pull/19257) | **next:** [#19251](https://github.com/DataDog/dd-trace-py/pull/19251) + +## Summary + +Native profiling py3.15 ABI (Echion frame state, cmake). + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + + +
+ +## 9/14 — `vlad/ddtracepy-315-profiling-collectors` → `vlad/ddtracepy-315-profiling-native` + +**Existing PR:** [#19251](https://github.com/DataDog/dd-trace-py/pull/19251) — set base to `vlad/ddtracepy-315-profiling-native`, mark draft, paste body from [`pr-bodies/09-vlad-ddtracepy-315-profiling-collectors.md`](pr-bodies/09-vlad-ddtracepy-315-profiling-collectors.md). + +**prev:** [#19250](https://github.com/DataDog/dd-trace-py/pull/19250) | **next:** [#19249](https://github.com/DataDog/dd-trace-py/pull/19249) + +
PR body (copy) + +**prev:** [#19250](https://github.com/DataDog/dd-trace-py/pull/19250) | **next:** [#19249](https://github.com/DataDog/dd-trace-py/pull/19249) + +## Summary + +Collector updates for 3.15. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + + +
+ +## 10/14 — `vlad/ddtracepy-315-profiling-only` → `vlad/ddtracepy-315-profiling-collectors` + +**Existing PR:** [#19249](https://github.com/DataDog/dd-trace-py/pull/19249) — set base to `vlad/ddtracepy-315-profiling-collectors`, mark draft, paste body from [`pr-bodies/10-vlad-ddtracepy-315-profiling-only.md`](pr-bodies/10-vlad-ddtracepy-315-profiling-only.md). + +**prev:** [#19251](https://github.com/DataDog/dd-trace-py/pull/19251) | **next:** [#19256](https://github.com/DataDog/dd-trace-py/pull/19256) + +
PR body (copy) + +**prev:** [#19251](https://github.com/DataDog/dd-trace-py/pull/19251) | **next:** [#19256](https://github.com/DataDog/dd-trace-py/pull/19256) + +## Summary + +Profiling CI matrix and setup.py gating. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + + +
+ +## 11/14 — `vlad/ddtracepy-315-profiling-asyncio-monitoring` → `vlad/ddtracepy-315-profiling-only` + +**Existing PR:** [#19256](https://github.com/DataDog/dd-trace-py/pull/19256) — set base to `vlad/ddtracepy-315-profiling-only`, mark draft, paste body from [`pr-bodies/11-vlad-ddtracepy-315-profiling-asyncio-monitoring.md`](pr-bodies/11-vlad-ddtracepy-315-profiling-asyncio-monitoring.md). + +**prev:** [#19249](https://github.com/DataDog/dd-trace-py/pull/19249) | **next:** [#19260](https://github.com/DataDog/dd-trace-py/pull/19260) + +
PR body (copy) + +**prev:** [#19249](https://github.com/DataDog/dd-trace-py/pull/19249) | **next:** [#19260](https://github.com/DataDog/dd-trace-py/pull/19260) + +## Summary + +Replace bytecode wrapping with `sys.monitoring` in `_asyncio.py`. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + + +
+ +## 12/14 — `vlad/315-profiling-dev-tooling` → `vlad/ddtracepy-315-profiling-asyncio-monitoring` + +**Existing PR:** [#19260](https://github.com/DataDog/dd-trace-py/pull/19260) — set base to `vlad/ddtracepy-315-profiling-asyncio-monitoring`, mark draft, paste body from [`pr-bodies/12-vlad-315-profiling-dev-tooling.md`](pr-bodies/12-vlad-315-profiling-dev-tooling.md). + +**prev:** [#19256](https://github.com/DataDog/dd-trace-py/pull/19256) | **next:** [#19258](https://github.com/DataDog/dd-trace-py/pull/19258) + +
PR body (copy) + +**prev:** [#19256](https://github.com/DataDog/dd-trace-py/pull/19256) | **next:** [#19258](https://github.com/DataDog/dd-trace-py/pull/19258) + +## Summary + +Profiling bring-up scripts, compatibility baselines, Echion migration runbook (GAP-03). + + +## Test plan + +- [ ] `scripts/run-profiling-tests --check-only` passes on 3.15 (when available) +- [ ] Docs build / link check + + +
+ +## 13/14 — `vlad/315-lib-injection-ssi` → `vlad/315-profiling-dev-tooling` + +**Existing PR:** [#19258](https://github.com/DataDog/dd-trace-py/pull/19258) — set base to `vlad/315-profiling-dev-tooling`, mark draft, paste body from [`pr-bodies/13-vlad-315-lib-injection-ssi.md`](pr-bodies/13-vlad-315-lib-injection-ssi.md). + +**prev:** [#19260](https://github.com/DataDog/dd-trace-py/pull/19260) | **next:** [#19259](https://github.com/DataDog/dd-trace-py/pull/19259) + +
PR body (copy) + +**prev:** [#19260](https://github.com/DataDog/dd-trace-py/pull/19260) | **next:** [#19259](https://github.com/DataDog/dd-trace-py/pull/19259) + +## Summary + +SSI allow-list + wheel download for 3.15 auto-instrumentation (GAP-01). Closes #17813. + + +## Test plan + +- [ ] lib-injection CI green + + +
+ +## 14/14 — `vlad/315-profiling-release-note` → `vlad/315-lib-injection-ssi` + +**Existing PR:** [#19259](https://github.com/DataDog/dd-trace-py/pull/19259) — set base to `vlad/315-lib-injection-ssi`, mark draft, paste body from [`pr-bodies/14-vlad-315-profiling-release-note.md`](pr-bodies/14-vlad-315-profiling-release-note.md). + +**prev:** [#19258](https://github.com/DataDog/dd-trace-py/pull/19258) | **next:** — + +
PR body (copy) + +**prev:** [#19258](https://github.com/DataDog/dd-trace-py/pull/19258) | **next:** — + +## Summary + +Customer-facing Reno fragment for profiling on 3.15 (GAP-02). + + +## Test plan + +- [ ] `riot run reno` validates fragment +- [ ] Merge with PR that lifts profiling native gate for 3.15 + + +
+ diff --git a/scripts/py315-stack/MANUAL_EXISTING_PRS.md b/scripts/py315-stack/MANUAL_EXISTING_PRS.md new file mode 100644 index 00000000000..99806d798a1 --- /dev/null +++ b/scripts/py315-stack/MANUAL_EXISTING_PRS.md @@ -0,0 +1,42 @@ +# Manual steps for the py3.15 stack PRs (EMU blocks `gh pr edit`) + +Branches are **rebased and force-pushed** (2026-07-23). Gab's commits are **unsigned** (not wrongly signed); +Vlad's commits are GPG-verified. `gh` cannot retarget bases or edit bodies from this EMU account — do the +steps below in the GitHub UI (or with a non-EMU token). + +## Close duplicate + +| PR | Action | +|----|--------| +| [#19248](https://github.com/DataDog/dd-trace-py/pull/19248) | **Close** — duplicate of [#17849](https://github.com/DataDog/dd-trace-py/pull/17849) (`gab/315-wrapping-context` ≡ `chore/315-wrapping-context`) | + +## Retarget base branch (required for incremental diffs) + +| # | PR | Head branch | Set base to | +|---|-----|-------------|-------------| +| 1 | [#19247](https://github.com/DataDog/dd-trace-py/pull/19247) | `gab/315-monitoring-multiplexer` | `main` | +| 2 | [#17849](https://github.com/DataDog/dd-trace-py/pull/17849) | `chore/315-wrapping-context` | `gab/315-monitoring-multiplexer` | +| 3 | [#19253](https://github.com/DataDog/dd-trace-py/pull/19253) | `vlad/315-ci-matrix` | `gab/315-wrapping-context` | +| 4 | [#19252](https://github.com/DataDog/dd-trace-py/pull/19252) | `vlad/315-ci-autoregen-lockfiles` | `vlad/315-ci-matrix` | +| 5 | [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) | `vlad/315-official-support` | `vlad/315-ci-autoregen-lockfiles` | +| 6 | [#19255](https://github.com/DataDog/dd-trace-py/pull/19255) | `vlad/315-peripheral-compat` | `vlad/315-official-support` | +| 7 | [#19257](https://github.com/DataDog/dd-trace-py/pull/19257) | `vlad/profiling-native-test-install-subdir` | `vlad/315-peripheral-compat` | +| 8 | [#19250](https://github.com/DataDog/dd-trace-py/pull/19250) | `vlad/ddtracepy-315-profiling-native` | `vlad/profiling-native-test-install-subdir` | +| 9 | [#19251](https://github.com/DataDog/dd-trace-py/pull/19251) | `vlad/ddtracepy-315-profiling-collectors` | `vlad/ddtracepy-315-profiling-native` | +| 10 | [#19249](https://github.com/DataDog/dd-trace-py/pull/19249) | `vlad/ddtracepy-315-profiling-only` | `vlad/ddtracepy-315-profiling-collectors` | +| 11 | [#19256](https://github.com/DataDog/dd-trace-py/pull/19256) | `vlad/ddtracepy-315-profiling-asyncio-monitoring` | `vlad/ddtracepy-315-profiling-only` | +| 12 | [#19260](https://github.com/DataDog/dd-trace-py/pull/19260) | `vlad/315-profiling-dev-tooling` | `vlad/ddtracepy-315-profiling-asyncio-monitoring` | +| 13 | [#19258](https://github.com/DataDog/dd-trace-py/pull/19258) | `vlad/315-lib-injection-ssi` | `vlad/315-profiling-dev-tooling` | +| 14 | [#19259](https://github.com/DataDog/dd-trace-py/pull/19259) | `vlad/315-profiling-release-note` | `vlad/315-lib-injection-ssi` | + +After retargeting, each PR should show **1 commit** (PR 11 shows **3** asyncio commits) vs its parent branch. + +## Update PR description + +Paste the body from `pr-bodies/NN-*.md` (first line: `**prev:** … | **next:** …`). Full index: [`DRAFT_PRS.md`](DRAFT_PRS.md). + +Numbers are in [`pr-numbers.env`](pr-numbers.env). Re-run `./generate-pr-urls.sh` after any number change. + +## Superseded closed PRs + +Do not reopen: #18488, #18503, #18504, #17624, #18389 — replaced by #19257–#19256 above. diff --git a/scripts/py315-stack/TRACKING.md b/scripts/py315-stack/TRACKING.md new file mode 100644 index 00000000000..e2aed0bcf14 --- /dev/null +++ b/scripts/py315-stack/TRACKING.md @@ -0,0 +1,79 @@ +# Python 3.15 — follow-up tracking (post-split) + +Tracks work **outside** the 11-PR stack that is still needed for full py3.15 parity. +Core profiling/tracing code lives in the stack; items here are deferred, optional polish, +or blocked on stack merge/CI. + +**Last updated:** 2026-07-22 (branches pushed locally — open PRs via compare links below) + +## Status summary + +| ID | Item | Priority | Status | Next step | +|----|------|----------|--------|-----------| +| GAP-01 | Lib-injection SSI ([#17977](https://github.com/DataDog/dd-trace-py/pull/17977)) | P1 | **Pushed** `vlad/315-lib-injection-ssi` | Open PR after stack merges | +| GAP-02 | Profiling release note | P1 | **Pushed** `vlad/315-profiling-release-note` | Merge with profiling ship PR | +| GAP-03 | Dev tooling + runbook | P2 | **Pushed** `vlad/315-profiling-dev-tooling` | Open PR now | +| GAP-04 | `test_uwsgi.py` type-ignore cleanup | P3 | **Won't fix** | mypy requires the ignores | + +### Open drafts (Prev/Next pre-filled in body) + +| # | Branch | PR body | Open draft | +|---|--------|---------|------------| +| 12 | `vlad/315-profiling-dev-tooling` | [`12-vlad-315-profiling-dev-tooling.md`](pr-bodies/12-vlad-315-profiling-dev-tooling.md) | [compare](https://github.com/DataDog/dd-trace-py/compare/vlad/ddtracepy-315-profiling-asyncio-monitoring...vlad/315-profiling-dev-tooling?expand=1&title=docs%28profiling%29%3A%20py3.15%20dev%20tooling%20and%20CPython%20upgrade%20runbook%20%28split%2012/14%29&body=%2A%2AStack%20navigation%3A%2A%2A%20%5B%E2%86%90%20Prev%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/18389%29%20%7C%20Next%20%E2%86%92%20_%28PR%20%2313%20%E2%80%94%20set%20PR13%20in%20pr-numbers.env%29_%0A%0A%23%23%20Summary%0A%0AProfiling%20bring-up%20scripts%20%28%60verify_profiler_compatibility%60%2C%20%60run-profiling-tests%60%29%2C%20compatibility%20baselines%2C%20and%20Echion%20migration%20runbook%20%28GAP-03%29.%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20%60scripts/run-profiling-tests%20--check-only%60%20passes%20on%203.15%20%28when%20available%29%0A-%20%5B%20%5D%20Docs%20build%20/%20link%20check%0A%0A&draft=1) | +| 13 | `vlad/315-lib-injection-ssi` | [`13-vlad-315-lib-injection-ssi.md`](pr-bodies/13-vlad-315-lib-injection-ssi.md) | [compare](https://github.com/DataDog/dd-trace-py/compare/vlad/ddtracepy-315-profiling-asyncio-monitoring...vlad/315-lib-injection-ssi?expand=1&title=chore%28py-315%29%3A%20enable%20lib-injection%20SSI%20for%20Python%203.15%20%28split%2013/14%29&body=%2A%2AStack%20navigation%3A%2A%2A%20%E2%86%90%20Prev%20_%28PR%20%2312%20%E2%80%94%20set%20PR12%20in%20pr-numbers.env%29_%20%7C%20Next%20%E2%86%92%20_%28PR%20%2314%20%E2%80%94%20set%20PR14%20in%20pr-numbers.env%29_%0A%0A%23%23%20Summary%0A%0ABump%20SSI%20allow-list%20and%20wheel%20download%20list%20for%203.15%20auto-instrumentation.%20Supersedes%20%2317977%20%28GAP-01%29.%20Merge%20after%20profiling%20stack.%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20lib-injection%20CI%20green%0A-%20%5B%20%5D%20Merge%20only%20after%20profiling%20natives%20build%20on%203.15%0A%0A&draft=1) | +| 14 | `vlad/315-profiling-release-note` | [`14-vlad-315-profiling-release-note.md`](pr-bodies/14-vlad-315-profiling-release-note.md) | [compare](https://github.com/DataDog/dd-trace-py/compare/vlad/ddtracepy-315-profiling-asyncio-monitoring...vlad/315-profiling-release-note?expand=1&title=docs%28releasenotes%29%3A%20profiling%20Python%203.15%20support%20note%20%28split%2014/14%29&body=%2A%2AStack%20navigation%3A%2A%2A%20%E2%86%90%20Prev%20_%28PR%20%2313%20%E2%80%94%20set%20PR13%20in%20pr-numbers.env%29_%20%7C%20Next%20%E2%86%92%0A%0A%23%23%20Summary%0A%0ACustomer-facing%20Reno%20fragment%20for%20profiling%20on%203.15%20%28GIL-enabled%3B%20free-threading%20not%20validated%29.%20Merge%20with%20profiling%20ship%20PR%20%28GAP-02%29.%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20%60riot%20run%20reno%60%20validates%20fragment%0A-%20%5B%20%5D%20Merge%20with%20PR%20that%20lifts%20profiling%20native%20gate%20for%203.15%0A%0A&draft=1) | + +After opening each PR, set `PR12`/`PR13`/`PR14` in [`pr-numbers.env`](pr-numbers.env) and re-run [`generate-pr-urls.sh`](generate-pr-urls.sh) to wire live Prev/Next links across the chain. + +--- + +## GAP-01 — Lib-injection SSI for Python 3.15 + +**Jira:** [PROF-14439](https://datadoghq.atlassian.net/browse/PROF-14439) +**Branch:** `vlad/315-lib-injection-ssi` (rebased on stack tip, cherry-pick of #17977) + +- [x] `lib-injection/sources/sitecustomize.py` — max `(3, 15)` → `(3, 16)` +- [x] `lib-injection/dl_wheels.py` — add `"3.15"` to `supported_versions` +- [ ] Open PR (merge **after** profiling stack); closes #17813 + +--- + +## GAP-02 — Profiling release note + +**Branch:** `vlad/315-profiling-release-note` +**File:** `releasenotes/notes/profiling-python315-support-5910a6a4e623716b.yaml` + +- [x] Reno fragment written (GIL-only; free-threading not validated) +- [ ] Merge with PR that lifts `setup.py` profiling native gate for 3.15 + +--- + +## GAP-03 — Profiling dev tooling + CPython upgrade runbook + +**Branch:** `vlad/315-profiling-dev-tooling` + +- [x] Port scripts + docs from `vlad/ddtracepy-upgrade-py-315-docs` +- [x] Link runbook from `docs/contributing.rst` + `docs/contributing-testing.rst` +- [ ] Open PR: `docs(profiling): py3.15 dev tooling and CPython upgrade runbook` + +--- + +## GAP-04 — `test_uwsgi.py` type-ignore cleanup + +**Won't fix** — removing ignores fails mypy (`attr-defined`, `unreachable`). + +--- + +## Stack hygiene (EMU blocks `gh pr create`) + +See [`DRAFT_PRS.md`](DRAFT_PRS.md): + +- [ ] Create draft PRs 1, 3–6 via compare URLs +- [ ] Retarget [#17849](https://github.com/DataDog/dd-trace-py/pull/17849) → `gab/315-monitoring-multiplexer` +- [ ] Reopen + retarget [#18488](https://github.com/DataDog/dd-trace-py/pull/18488)–[#18389](https://github.com/DataDog/dd-trace-py/pull/18389) + +--- + +## Archived PRs — no further audit + +#17446, #17055, #17531, #17532, #17294, #17295, #17596, #17730, #18146, #17978, #17977, #17847, #17792 diff --git a/scripts/py315-stack/generate-pr-urls.sh b/scripts/py315-stack/generate-pr-urls.sh new file mode 100755 index 00000000000..7ad4d7b176e --- /dev/null +++ b/scripts/py315-stack/generate-pr-urls.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Generate GitHub compare URLs + PR bodies with prev/next navigation for the py3.15 stack. +# EMU blocks `gh pr create`; use open-all-draft-prs.sh or compare URLs in DRAFT_PRS.md. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOT" + +ENV_FILE="$(dirname "$0")/pr-numbers.env" +# shellcheck disable=SC1090 +source "$ENV_FILE" + +STACK_MAX=14 +REPO="DataDog/dd-trace-py" +BASE_URL="https://github.com/${REPO}" + +pr_num() { + local n=$1 + local var="PR${n}" + echo "${!var:-}" +} + +pr_url() { + local n=$1 + local num + num="$(pr_num "$n")" + if [[ -n "$num" ]]; then + echo "${BASE_URL}/pull/${num}" + else + echo "" + fi +} + +nav_line() { + local idx=$1 + local prev_n=$((idx - 1)) + local next_n=$((idx + 1)) + local prev next prev_num next_num prev_url next_url + + if [[ $prev_n -ge 1 ]]; then + prev_num="$(pr_num "$prev_n")" + prev_url="$(pr_url "$prev_n")" + if [[ -n "$prev_num" && -n "$prev_url" ]]; then + prev="[#${prev_num}](${prev_url})" + else + prev="_#${prev_n} (pending)_" + fi + else + prev="—" + fi + + if [[ $next_n -le $STACK_MAX ]]; then + next_num="$(pr_num "$next_n")" + next_url="$(pr_url "$next_n")" + if [[ -n "$next_num" && -n "$next_url" ]]; then + next="[#${next_num}](${next_url})" + else + next="_#${next_n} (pending)_" + fi + else + next="—" + fi + + echo "**prev:** ${prev} | **next:** ${next}" +} + +urlencode() { + python3 -c 'import urllib.parse,sys; print(urllib.parse.quote(sys.stdin.read()))' +} + +write_pr_entry() { + local idx=$1 + local total=$2 + local head=$3 + local base=$4 + local title=$5 + local summary=$6 + local action_note=${7:-} + local test_plan=${8:-"- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch"} + + local nav body_file body_encoded title_encoded compare_url existing pr_var existing_num + + nav="$(nav_line "$idx")" + body_file="${OUT_DIR}/$(printf '%02d' "$idx")-${head//\//-}.md" + + cat > "$body_file" <PR body (copy)" + echo "" + cat "$body_file" + echo "" + echo "" + echo "" + } >> "$INDEX" + + printf '%s\n' "$compare_url" > "${OUT_DIR}/$(printf '%02d' "$idx")-compare.url" +} + +# idx|head|base|title|summary|action_note (optional, use · for empty) +ENTRIES=( + "1|gab/315-monitoring-multiplexer|main|feat(internal): sys.monitoring multiplexer for Python 3.15 (split 1/14)|Introduces \`ddtrace.internal.monitoring\` — shared \`sys.monitoring\` multiplexer (Gab). Split from #17849.|·" + "2|chore/315-wrapping-context|gab/315-monitoring-multiplexer|chore(wrapping): Python 3.15 wrapping context support (split 2/14)|Wrapping context + bytecode_injection for 3.15 (Gab + await/send fix).|Retarget existing **#17849** to base \`gab/315-monitoring-multiplexer\`." + "3|vlad/315-ci-matrix|gab/315-wrapping-context|ci(py3.15): add 3.15 to riot matrix with gated suites (split 3/14)|Riotfile 3.15, docker testrunner, suite gating.|·" + "4|vlad/315-ci-autoregen-lockfiles|vlad/315-ci-matrix|ci: auto-commit regenerated riot lockfiles on PR branches (split 4/14)|Self-healing lockfile drift on PR branches.|·" + "5|vlad/315-official-support|vlad/315-ci-autoregen-lockfiles|chore(py3.15): declare official 3.15 support in packaging (split 5/14)|pyproject.toml, requirements.csv, riot lockfiles.|·" + "6|vlad/315-peripheral-compat|vlad/315-official-support|fix(py3.15): peripheral compat for profiling, logging, appsec (split 6/14)|Graceful degradation + test skips outside wrapping core.|·" + "7|vlad/profiling-native-test-install-subdir|vlad/315-peripheral-compat|chore(profiling): native test install subdirs (PROF-14200) (split 7/14)|INSTALL_SUBDIR for py3.15 native tests.|·" + "8|vlad/ddtracepy-315-profiling-native|vlad/profiling-native-test-install-subdir|chore(profiling): native C++/Rust py3.15 ABI support (split 8/14)|Native profiling py3.15 ABI (Echion frame state, cmake).|·" + "9|vlad/ddtracepy-315-profiling-collectors|vlad/ddtracepy-315-profiling-native|chore(profiling): update Python profiling collectors for py3.15 (split 9/14)|Collector updates for 3.15.|·" + "10|vlad/ddtracepy-315-profiling-only|vlad/ddtracepy-315-profiling-collectors|ci(profiling): wire py3.15 into build matrix and CI (split 10/14)|Profiling CI matrix and setup.py gating.|·" + "11|vlad/ddtracepy-315-profiling-asyncio-monitoring|vlad/ddtracepy-315-profiling-only|refactor(profiling): sys.monitoring asyncio path for py3.15 (split 11/14)|Replace bytecode wrapping with \`sys.monitoring\` in \`_asyncio.py\`.|·" + "12|vlad/315-profiling-dev-tooling|vlad/ddtracepy-315-profiling-asyncio-monitoring|docs(profiling): py3.15 dev tooling and CPython upgrade runbook (split 12/14)|Profiling bring-up scripts, compatibility baselines, Echion migration runbook (GAP-03).|·" + "13|vlad/315-lib-injection-ssi|vlad/315-profiling-dev-tooling|chore(py-315): enable lib-injection SSI for Python 3.15 (split 13/14)|SSI allow-list + wheel download for 3.15 auto-instrumentation (GAP-01). Closes #17813.|·" + "14|vlad/315-profiling-release-note|vlad/315-lib-injection-ssi|docs(releasenotes): profiling Python 3.15 support note (split 14/14)|Customer-facing Reno fragment for profiling on 3.15 (GAP-02).|·" +) + +OUT_DIR="$(dirname "$0")/pr-bodies" +mkdir -p "$OUT_DIR" +INDEX="$(dirname "$0")/DRAFT_PRS.md" + +{ + echo "# Python 3.15 stack — draft PRs (1–14)" + echo "" + echo "EMU blocks \`gh pr create\` / \`gh pr edit\`. Run [\`sync-pr-numbers.sh\`](sync-pr-numbers.sh) then [\`open-all-draft-prs.sh\`](open-all-draft-prs.sh)." + echo "Set numbers: \`./sync-pr-numbers.sh PR1=19250 PR6=19255 ...\` (preserves existing, fills gaps from GitHub)." + echo "" +} > "$INDEX" + +for entry in "${ENTRIES[@]}"; do + IFS='|' read -r idx head base title summary action_note <<< "$entry" + if [[ "$action_note" == "·" ]]; then action_note=""; else + action_note=$'\n\n'"${action_note}" + fi + + test_plan="- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch" + case "$idx" in + 12) test_plan="- [ ] \`scripts/run-profiling-tests --check-only\` passes on 3.15 (when available) +- [ ] Docs build / link check" ;; + 13) test_plan="- [ ] lib-injection CI green" ;; + 14) test_plan="- [ ] \`riot run reno\` validates fragment +- [ ] Merge with PR that lifts profiling native gate for 3.15" ;; + esac + + write_pr_entry "$idx" "$STACK_MAX" "$head" "$base" "$title" "$summary" "$action_note" "$test_plan" +done + +echo "Wrote ${INDEX} and ${OUT_DIR}/*.md" diff --git a/scripts/py315-stack/open-all-draft-prs.sh b/scripts/py315-stack/open-all-draft-prs.sh new file mode 100755 index 00000000000..f78e271dcad --- /dev/null +++ b/scripts/py315-stack/open-all-draft-prs.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Open GitHub compare URLs to create draft PRs in stack order (1→14). +# Skips entries that already have a PR number in pr-numbers.env. +set -euo pipefail + +DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck disable=SC1090 +source "$DIR/pr-numbers.env" + +"$DIR/generate-pr-urls.sh" >/dev/null + +opened=0 +for i in $(seq 1 14); do + var="PR${i}" + num="${!var:-}" + if [[ -n "$num" ]]; then + echo "SKIP ${i}/14 — PR #${num} already exists" + continue + fi + url_file="${DIR}/pr-bodies/$(printf '%02d' "$i")-compare.url" + if [[ ! -f "$url_file" ]]; then + echo "WARN: missing ${url_file}" >&2 + continue + fi + url="$(cat "$url_file")" + echo "OPEN ${i}/14 — ${url}" + if command -v open >/dev/null 2>&1; then + open "$url" + opened=$((opened + 1)) + # Avoid browser tab flood; pause between opens. + sleep 2 + else + echo "$url" + fi +done + +if [[ $opened -eq 0 ]] && command -v open >/dev/null 2>&1; then + echo "No compare URLs opened (all PR slots filled or open(1) unavailable)." + echo "Compare links: ${DIR}/DRAFT_PRS.md" +fi diff --git a/scripts/py315-stack/pr-bodies/01-compare.url b/scripts/py315-stack/pr-bodies/01-compare.url new file mode 100644 index 00000000000..612c362117b --- /dev/null +++ b/scripts/py315-stack/pr-bodies/01-compare.url @@ -0,0 +1 @@ +https://github.com/DataDog/dd-trace-py/compare/main...gab/315-monitoring-multiplexer?expand=1&title=feat%28internal%29%3A%20sys.monitoring%20multiplexer%20for%20Python%203.15%20%28split%201/14%29&body=%2A%2Aprev%3A%2A%2A%20%E2%80%94%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2317849%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/17849%29%0A%0A%23%23%20Summary%0A%0AIntroduces%20%60ddtrace.internal.monitoring%60%20%E2%80%94%20shared%20%60sys.monitoring%60%20multiplexer%20%28Gab%29.%20Split%20from%20%2317849.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/01-gab-315-monitoring-multiplexer.md b/scripts/py315-stack/pr-bodies/01-gab-315-monitoring-multiplexer.md new file mode 100644 index 00000000000..b92ce78979b --- /dev/null +++ b/scripts/py315-stack/pr-bodies/01-gab-315-monitoring-multiplexer.md @@ -0,0 +1,12 @@ +**prev:** — | **next:** [#17849](https://github.com/DataDog/dd-trace-py/pull/17849) + +## Summary + +Introduces `ddtrace.internal.monitoring` — shared `sys.monitoring` multiplexer (Gab). Split from #17849. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/02-chore-315-wrapping-context.md b/scripts/py315-stack/pr-bodies/02-chore-315-wrapping-context.md new file mode 100644 index 00000000000..b3dbae5a9eb --- /dev/null +++ b/scripts/py315-stack/pr-bodies/02-chore-315-wrapping-context.md @@ -0,0 +1,14 @@ +**prev:** [#19247](https://github.com/DataDog/dd-trace-py/pull/19247) | **next:** [#19253](https://github.com/DataDog/dd-trace-py/pull/19253) + +## Summary + +Wrapping context + bytecode_injection for 3.15 (Gab + await/send fix). + + +Retarget existing **#17849** to base `gab/315-monitoring-multiplexer`. + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/02-compare.url b/scripts/py315-stack/pr-bodies/02-compare.url new file mode 100644 index 00000000000..3a64f08ecc5 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/02-compare.url @@ -0,0 +1 @@ +https://github.com/DataDog/dd-trace-py/compare/gab/315-monitoring-multiplexer...chore/315-wrapping-context?expand=1&title=chore%28wrapping%29%3A%20Python%203.15%20wrapping%20context%20support%20%28split%202/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319247%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19247%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319253%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19253%29%0A%0A%23%23%20Summary%0A%0AWrapping%20context%20%2B%20bytecode_injection%20for%203.15%20%28Gab%20%2B%20await/send%20fix%29.%0A%0A%0ARetarget%20existing%20%2A%2A%2317849%2A%2A%20to%20base%20%60gab/315-monitoring-multiplexer%60.%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/03-compare.url b/scripts/py315-stack/pr-bodies/03-compare.url new file mode 100644 index 00000000000..5498f72d642 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/03-compare.url @@ -0,0 +1 @@ +https://github.com/DataDog/dd-trace-py/compare/gab/315-wrapping-context...vlad/315-ci-matrix?expand=1&title=ci%28py3.15%29%3A%20add%203.15%20to%20riot%20matrix%20with%20gated%20suites%20%28split%203/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2317849%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/17849%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319252%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19252%29%0A%0A%23%23%20Summary%0A%0ARiotfile%203.15%2C%20docker%20testrunner%2C%20suite%20gating.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/03-vlad-315-ci-matrix.md b/scripts/py315-stack/pr-bodies/03-vlad-315-ci-matrix.md new file mode 100644 index 00000000000..08dbb848a65 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/03-vlad-315-ci-matrix.md @@ -0,0 +1,12 @@ +**prev:** [#17849](https://github.com/DataDog/dd-trace-py/pull/17849) | **next:** [#19252](https://github.com/DataDog/dd-trace-py/pull/19252) + +## Summary + +Riotfile 3.15, docker testrunner, suite gating. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/04-compare.url b/scripts/py315-stack/pr-bodies/04-compare.url new file mode 100644 index 00000000000..0560d1a8130 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/04-compare.url @@ -0,0 +1 @@ +https://github.com/DataDog/dd-trace-py/compare/vlad/315-ci-matrix...vlad/315-ci-autoregen-lockfiles?expand=1&title=ci%3A%20auto-commit%20regenerated%20riot%20lockfiles%20on%20PR%20branches%20%28split%204/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319253%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19253%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319254%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19254%29%0A%0A%23%23%20Summary%0A%0ASelf-healing%20lockfile%20drift%20on%20PR%20branches.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/04-vlad-315-ci-autoregen-lockfiles.md b/scripts/py315-stack/pr-bodies/04-vlad-315-ci-autoregen-lockfiles.md new file mode 100644 index 00000000000..212ee83b81b --- /dev/null +++ b/scripts/py315-stack/pr-bodies/04-vlad-315-ci-autoregen-lockfiles.md @@ -0,0 +1,12 @@ +**prev:** [#19253](https://github.com/DataDog/dd-trace-py/pull/19253) | **next:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) + +## Summary + +Self-healing lockfile drift on PR branches. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/05-compare.url b/scripts/py315-stack/pr-bodies/05-compare.url new file mode 100644 index 00000000000..8e2903e94be --- /dev/null +++ b/scripts/py315-stack/pr-bodies/05-compare.url @@ -0,0 +1 @@ +https://github.com/DataDog/dd-trace-py/compare/vlad/315-ci-autoregen-lockfiles...vlad/315-official-support?expand=1&title=chore%28py3.15%29%3A%20declare%20official%203.15%20support%20in%20packaging%20%28split%205/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319252%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19252%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319255%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19255%29%0A%0A%23%23%20Summary%0A%0Apyproject.toml%2C%20requirements.csv%2C%20riot%20lockfiles.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/05-vlad-315-official-support.md b/scripts/py315-stack/pr-bodies/05-vlad-315-official-support.md new file mode 100644 index 00000000000..0ef353b1a29 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/05-vlad-315-official-support.md @@ -0,0 +1,12 @@ +**prev:** [#19252](https://github.com/DataDog/dd-trace-py/pull/19252) | **next:** [#19255](https://github.com/DataDog/dd-trace-py/pull/19255) + +## Summary + +pyproject.toml, requirements.csv, riot lockfiles. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/06-compare.url b/scripts/py315-stack/pr-bodies/06-compare.url new file mode 100644 index 00000000000..24c7f5d796a --- /dev/null +++ b/scripts/py315-stack/pr-bodies/06-compare.url @@ -0,0 +1 @@ +https://github.com/DataDog/dd-trace-py/compare/vlad/315-official-support...vlad/315-peripheral-compat?expand=1&title=fix%28py3.15%29%3A%20peripheral%20compat%20for%20profiling%2C%20logging%2C%20appsec%20%28split%206/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319254%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19254%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319257%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19257%29%0A%0A%23%23%20Summary%0A%0AGraceful%20degradation%20%2B%20test%20skips%20outside%20wrapping%20core.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/06-vlad-315-peripheral-compat.md b/scripts/py315-stack/pr-bodies/06-vlad-315-peripheral-compat.md new file mode 100644 index 00000000000..61201fc2069 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/06-vlad-315-peripheral-compat.md @@ -0,0 +1,12 @@ +**prev:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) | **next:** [#19257](https://github.com/DataDog/dd-trace-py/pull/19257) + +## Summary + +Graceful degradation + test skips outside wrapping core. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/07-compare.url b/scripts/py315-stack/pr-bodies/07-compare.url new file mode 100644 index 00000000000..89ac5080617 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/07-compare.url @@ -0,0 +1 @@ +https://github.com/DataDog/dd-trace-py/compare/vlad/315-peripheral-compat...vlad/profiling-native-test-install-subdir?expand=1&title=chore%28profiling%29%3A%20native%20test%20install%20subdirs%20%28PROF-14200%29%20%28split%207/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319255%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19255%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319250%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19250%29%0A%0A%23%23%20Summary%0A%0AINSTALL_SUBDIR%20for%20py3.15%20native%20tests.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/07-vlad-profiling-native-test-install-subdir.md b/scripts/py315-stack/pr-bodies/07-vlad-profiling-native-test-install-subdir.md new file mode 100644 index 00000000000..2968c6b5ce5 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/07-vlad-profiling-native-test-install-subdir.md @@ -0,0 +1,12 @@ +**prev:** [#19255](https://github.com/DataDog/dd-trace-py/pull/19255) | **next:** [#19250](https://github.com/DataDog/dd-trace-py/pull/19250) + +## Summary + +INSTALL_SUBDIR for py3.15 native tests. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/08-compare.url b/scripts/py315-stack/pr-bodies/08-compare.url new file mode 100644 index 00000000000..71311ea47f2 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/08-compare.url @@ -0,0 +1 @@ +https://github.com/DataDog/dd-trace-py/compare/vlad/profiling-native-test-install-subdir...vlad/ddtracepy-315-profiling-native?expand=1&title=chore%28profiling%29%3A%20native%20C%2B%2B/Rust%20py3.15%20ABI%20support%20%28split%208/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319257%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19257%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319251%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19251%29%0A%0A%23%23%20Summary%0A%0ANative%20profiling%20py3.15%20ABI%20%28Echion%20frame%20state%2C%20cmake%29.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/08-vlad-ddtracepy-315-profiling-native.md b/scripts/py315-stack/pr-bodies/08-vlad-ddtracepy-315-profiling-native.md new file mode 100644 index 00000000000..5fe5938e23a --- /dev/null +++ b/scripts/py315-stack/pr-bodies/08-vlad-ddtracepy-315-profiling-native.md @@ -0,0 +1,12 @@ +**prev:** [#19257](https://github.com/DataDog/dd-trace-py/pull/19257) | **next:** [#19251](https://github.com/DataDog/dd-trace-py/pull/19251) + +## Summary + +Native profiling py3.15 ABI (Echion frame state, cmake). + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/09-compare.url b/scripts/py315-stack/pr-bodies/09-compare.url new file mode 100644 index 00000000000..11c123e65c2 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/09-compare.url @@ -0,0 +1 @@ +https://github.com/DataDog/dd-trace-py/compare/vlad/ddtracepy-315-profiling-native...vlad/ddtracepy-315-profiling-collectors?expand=1&title=chore%28profiling%29%3A%20update%20Python%20profiling%20collectors%20for%20py3.15%20%28split%209/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319250%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19250%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319249%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19249%29%0A%0A%23%23%20Summary%0A%0ACollector%20updates%20for%203.15.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/09-vlad-ddtracepy-315-profiling-collectors.md b/scripts/py315-stack/pr-bodies/09-vlad-ddtracepy-315-profiling-collectors.md new file mode 100644 index 00000000000..e5757c61a51 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/09-vlad-ddtracepy-315-profiling-collectors.md @@ -0,0 +1,12 @@ +**prev:** [#19250](https://github.com/DataDog/dd-trace-py/pull/19250) | **next:** [#19249](https://github.com/DataDog/dd-trace-py/pull/19249) + +## Summary + +Collector updates for 3.15. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/10-compare.url b/scripts/py315-stack/pr-bodies/10-compare.url new file mode 100644 index 00000000000..34007c29a4d --- /dev/null +++ b/scripts/py315-stack/pr-bodies/10-compare.url @@ -0,0 +1 @@ +https://github.com/DataDog/dd-trace-py/compare/vlad/ddtracepy-315-profiling-collectors...vlad/ddtracepy-315-profiling-only?expand=1&title=ci%28profiling%29%3A%20wire%20py3.15%20into%20build%20matrix%20and%20CI%20%28split%2010/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319251%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19251%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319256%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19256%29%0A%0A%23%23%20Summary%0A%0AProfiling%20CI%20matrix%20and%20setup.py%20gating.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/10-vlad-ddtracepy-315-profiling-only.md b/scripts/py315-stack/pr-bodies/10-vlad-ddtracepy-315-profiling-only.md new file mode 100644 index 00000000000..41772cb5faf --- /dev/null +++ b/scripts/py315-stack/pr-bodies/10-vlad-ddtracepy-315-profiling-only.md @@ -0,0 +1,12 @@ +**prev:** [#19251](https://github.com/DataDog/dd-trace-py/pull/19251) | **next:** [#19256](https://github.com/DataDog/dd-trace-py/pull/19256) + +## Summary + +Profiling CI matrix and setup.py gating. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/11-compare.url b/scripts/py315-stack/pr-bodies/11-compare.url new file mode 100644 index 00000000000..889cb6c6e56 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/11-compare.url @@ -0,0 +1 @@ +https://github.com/DataDog/dd-trace-py/compare/vlad/ddtracepy-315-profiling-only...vlad/ddtracepy-315-profiling-asyncio-monitoring?expand=1&title=refactor%28profiling%29%3A%20sys.monitoring%20asyncio%20path%20for%20py3.15%20%28split%2011/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319249%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19249%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319260%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19260%29%0A%0A%23%23%20Summary%0A%0AReplace%20bytecode%20wrapping%20with%20%60sys.monitoring%60%20in%20%60_asyncio.py%60.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/11-vlad-ddtracepy-315-profiling-asyncio-monitoring.md b/scripts/py315-stack/pr-bodies/11-vlad-ddtracepy-315-profiling-asyncio-monitoring.md new file mode 100644 index 00000000000..15034fa6f14 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/11-vlad-ddtracepy-315-profiling-asyncio-monitoring.md @@ -0,0 +1,12 @@ +**prev:** [#19249](https://github.com/DataDog/dd-trace-py/pull/19249) | **next:** [#19260](https://github.com/DataDog/dd-trace-py/pull/19260) + +## Summary + +Replace bytecode wrapping with `sys.monitoring` in `_asyncio.py`. + + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/12-compare.url b/scripts/py315-stack/pr-bodies/12-compare.url new file mode 100644 index 00000000000..a8240096826 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/12-compare.url @@ -0,0 +1 @@ +https://github.com/DataDog/dd-trace-py/compare/vlad/ddtracepy-315-profiling-asyncio-monitoring...vlad/315-profiling-dev-tooling?expand=1&title=docs%28profiling%29%3A%20py3.15%20dev%20tooling%20and%20CPython%20upgrade%20runbook%20%28split%2012/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319256%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19256%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319258%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19258%29%0A%0A%23%23%20Summary%0A%0AProfiling%20bring-up%20scripts%2C%20compatibility%20baselines%2C%20Echion%20migration%20runbook%20%28GAP-03%29.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20%60scripts/run-profiling-tests%20--check-only%60%20passes%20on%203.15%20%28when%20available%29%0A-%20%5B%20%5D%20Docs%20build%20/%20link%20check%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/12-vlad-315-profiling-dev-tooling.md b/scripts/py315-stack/pr-bodies/12-vlad-315-profiling-dev-tooling.md new file mode 100644 index 00000000000..b75cb58efed --- /dev/null +++ b/scripts/py315-stack/pr-bodies/12-vlad-315-profiling-dev-tooling.md @@ -0,0 +1,12 @@ +**prev:** [#19256](https://github.com/DataDog/dd-trace-py/pull/19256) | **next:** [#19258](https://github.com/DataDog/dd-trace-py/pull/19258) + +## Summary + +Profiling bring-up scripts, compatibility baselines, Echion migration runbook (GAP-03). + + +## Test plan + +- [ ] `scripts/run-profiling-tests --check-only` passes on 3.15 (when available) +- [ ] Docs build / link check + diff --git a/scripts/py315-stack/pr-bodies/13-compare.url b/scripts/py315-stack/pr-bodies/13-compare.url new file mode 100644 index 00000000000..4fae50ae9cc --- /dev/null +++ b/scripts/py315-stack/pr-bodies/13-compare.url @@ -0,0 +1 @@ +https://github.com/DataDog/dd-trace-py/compare/vlad/315-profiling-dev-tooling...vlad/315-lib-injection-ssi?expand=1&title=chore%28py-315%29%3A%20enable%20lib-injection%20SSI%20for%20Python%203.15%20%28split%2013/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319260%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19260%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319259%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19259%29%0A%0A%23%23%20Summary%0A%0ASSI%20allow-list%20%2B%20wheel%20download%20for%203.15%20auto-instrumentation%20%28GAP-01%29.%20Closes%20%2317813.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20lib-injection%20CI%20green%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/13-vlad-315-lib-injection-ssi.md b/scripts/py315-stack/pr-bodies/13-vlad-315-lib-injection-ssi.md new file mode 100644 index 00000000000..02bae9b842b --- /dev/null +++ b/scripts/py315-stack/pr-bodies/13-vlad-315-lib-injection-ssi.md @@ -0,0 +1,11 @@ +**prev:** [#19260](https://github.com/DataDog/dd-trace-py/pull/19260) | **next:** [#19259](https://github.com/DataDog/dd-trace-py/pull/19259) + +## Summary + +SSI allow-list + wheel download for 3.15 auto-instrumentation (GAP-01). Closes #17813. + + +## Test plan + +- [ ] lib-injection CI green + diff --git a/scripts/py315-stack/pr-bodies/14-compare.url b/scripts/py315-stack/pr-bodies/14-compare.url new file mode 100644 index 00000000000..0bfe2fe16bd --- /dev/null +++ b/scripts/py315-stack/pr-bodies/14-compare.url @@ -0,0 +1 @@ +https://github.com/DataDog/dd-trace-py/compare/vlad/315-lib-injection-ssi...vlad/315-profiling-release-note?expand=1&title=docs%28releasenotes%29%3A%20profiling%20Python%203.15%20support%20note%20%28split%2014/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319258%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19258%29%20%7C%20%2A%2Anext%3A%2A%2A%20%E2%80%94%0A%0A%23%23%20Summary%0A%0ACustomer-facing%20Reno%20fragment%20for%20profiling%20on%203.15%20%28GAP-02%29.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20%60riot%20run%20reno%60%20validates%20fragment%0A-%20%5B%20%5D%20Merge%20with%20PR%20that%20lifts%20profiling%20native%20gate%20for%203.15%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/14-vlad-315-profiling-release-note.md b/scripts/py315-stack/pr-bodies/14-vlad-315-profiling-release-note.md new file mode 100644 index 00000000000..d93a96deffd --- /dev/null +++ b/scripts/py315-stack/pr-bodies/14-vlad-315-profiling-release-note.md @@ -0,0 +1,12 @@ +**prev:** [#19258](https://github.com/DataDog/dd-trace-py/pull/19258) | **next:** — + +## Summary + +Customer-facing Reno fragment for profiling on 3.15 (GAP-02). + + +## Test plan + +- [ ] `riot run reno` validates fragment +- [ ] Merge with PR that lifts profiling native gate for 3.15 + diff --git a/scripts/py315-stack/pr-numbers.env b/scripts/py315-stack/pr-numbers.env new file mode 100644 index 00000000000..f12ffe155b9 --- /dev/null +++ b/scripts/py315-stack/pr-numbers.env @@ -0,0 +1,15 @@ +# PR numbers for stack navigation. ./sync-pr-numbers.sh [PRn=num ...] then regenerates bodies. +PR1=19247 +PR2=17849 +PR3=19253 +PR4=19252 +PR5=19254 +PR6=19255 +PR7=19257 +PR8=19250 +PR9=19251 +PR10=19249 +PR11=19256 +PR12=19260 +PR13=19258 +PR14=19259 diff --git a/scripts/py315-stack/rebuild-stack.sh b/scripts/py315-stack/rebuild-stack.sh new file mode 100755 index 00000000000..2c7c7d9ea56 --- /dev/null +++ b/scripts/py315-stack/rebuild-stack.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# Rebuild the 14-PR stack off origin/main with correct incremental commits. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOT" + +git fetch origin + +GAB="Gabriele N. Tornetta " +SRC=gab/315-monitoring-multiplexer + +rebuild_layer() { + local branch=$1 + local parent=$2 + local author_kind=$3 + local msg=$4 + local src=${5:-$branch} + + echo "==> ${branch} (from ${src}, parent ${parent})" + git checkout -B _stack-fix "${parent}" + files=() + while IFS= read -r f; do files+=("$f"); done < <(git diff --name-only "${parent}" "origin/${src}") + if [[ ${#files[@]} -eq 0 ]]; then + echo "No file diff for ${src}; skipping commit" >&2 + return 1 + fi + git checkout "origin/${src}" -- "${files[@]}" + if [[ "$author_kind" == gab ]]; then + git -c commit.gpgsign=false commit --author="$GAB" -m "$msg" + else + git commit -m "$msg" + fi + git branch -f "$branch" HEAD +} + +git checkout -B _stack-fix origin/main + +# PR1 +git checkout origin/gab/315-monitoring-multiplexer -- \ + ddtrace/internal/monitoring.py tests/internal/test_monitoring.py +git -c commit.gpgsign=false commit --author="$GAB" -m "feat(internal): add sys.monitoring multiplexer for Python 3.15 + +Introduces ddtrace.internal.monitoring for Python 3.15+. + +Part of the #17849 split (PR 1/14)." +git branch -f gab/315-monitoring-multiplexer HEAD + +# PR2 +rebuild_layer chore/315-wrapping-context gab/315-monitoring-multiplexer gab \ + "chore(wrapping): add Python 3.15 wrapping context support + +Part of the #17849 split (PR 2/14)." chore/315-wrapping-context +git branch -f gab/315-wrapping-context HEAD + +rebuild_layer vlad/315-ci-matrix chore/315-wrapping-context vlad \ + "ci(py3.15): add 3.15 to riot matrix with gated suites + +Part of the #17849 split (PR 3/14)." +rebuild_layer vlad/315-ci-autoregen-lockfiles vlad/315-ci-matrix vlad \ + "ci: auto-commit regenerated riot lockfiles on PR branches + +Part of the #17849 split (PR 4/14)." +rebuild_layer vlad/315-official-support vlad/315-ci-autoregen-lockfiles vlad \ + "chore(py3.15): declare official 3.15 support in packaging + +Part of the #17849 split (PR 5/14)." +rebuild_layer vlad/315-peripheral-compat vlad/315-official-support vlad \ + "fix(py3.15): peripheral compat for profiling, logging, and appsec + +Part of the #17849 split (PR 6/14)." +rebuild_layer vlad/profiling-native-test-install-subdir vlad/315-peripheral-compat vlad \ + "feat(profiling): add INSTALL_SUBDIR keyword to dd_wrapper_add_test + +Part of the profiling stack (PR 7/14)." +rebuild_layer vlad/ddtracepy-315-profiling-native vlad/profiling-native-test-install-subdir vlad \ + "chore(profiling): native C++/Rust py3.15 ABI support + +Part of the profiling stack (PR 8/14)." +rebuild_layer vlad/ddtracepy-315-profiling-collectors vlad/ddtracepy-315-profiling-native vlad \ + "chore(profiling): update Python profiling collectors for py3.15 + +Part of the profiling stack (PR 9/14)." +rebuild_layer vlad/ddtracepy-315-profiling-only vlad/ddtracepy-315-profiling-collectors vlad \ + "ci(profiling): wire py3.15 into build matrix, riotfile, and CI + +Part of the profiling stack (PR 10/14)." + +# PR11: squash 3 asyncio commits into one layer diff +files=() +while IFS= read -r f; do files+=("$f"); done < <(git diff --name-only vlad/ddtracepy-315-profiling-only origin/vlad/ddtracepy-315-profiling-asyncio-monitoring) +git checkout -B _stack-fix vlad/ddtracepy-315-profiling-only +git checkout origin/vlad/ddtracepy-315-profiling-asyncio-monitoring -- "${files[@]}" +git commit -m "refactor(profiling): sys.monitoring asyncio path for py3.15 + +Part of the profiling stack (PR 11/14)." +git branch -f vlad/ddtracepy-315-profiling-asyncio-monitoring HEAD + +rebuild_layer vlad/315-profiling-dev-tooling vlad/ddtracepy-315-profiling-asyncio-monitoring vlad \ + "docs(profiling): add py3.15 dev tooling and CPython upgrade runbook + +Part of the post-stack follow-ups (PR 12/14)." + +# PR12 tooling scripts (from origin dev-tooling branch) +git checkout origin/vlad/315-profiling-dev-tooling -- scripts/py315-stack/ 2>/dev/null || true +if [[ -d scripts/py315-stack ]]; then + git add scripts/py315-stack/ + if ! git diff --cached --quiet; then + git commit -m "chore(py315): stack PR navigation scripts and bodies + +Part of the post-stack follow-ups (PR 12/14)." + fi +fi +git branch -f vlad/315-profiling-dev-tooling HEAD + +rebuild_layer vlad/315-lib-injection-ssi vlad/315-profiling-dev-tooling vlad \ + "chore(py-315): enable lib-injection SSI for Python 3.15 + +Part of the post-stack follow-ups (PR 13/14)." +rebuild_layer vlad/315-profiling-release-note vlad/315-lib-injection-ssi vlad \ + "docs(releasenotes): add profiling Python 3.15 support note + +Part of the post-stack follow-ups (PR 14/14)." + +echo "" +echo "Stack rebuilt. Branch tips:" +for b in gab/315-monitoring-multiplexer chore/315-wrapping-context vlad/315-ci-matrix \ + vlad/315-ci-autoregen-lockfiles vlad/315-official-support vlad/315-peripheral-compat \ + vlad/profiling-native-test-install-subdir vlad/ddtracepy-315-profiling-native \ + vlad/ddtracepy-315-profiling-collectors vlad/ddtracepy-315-profiling-only \ + vlad/ddtracepy-315-profiling-asyncio-monitoring vlad/315-profiling-dev-tooling \ + vlad/315-lib-injection-ssi vlad/315-profiling-release-note; do + inc=$(git rev-list --count "origin/${b}^..${b}" 2>/dev/null || git rev-list --count "${b}^..${b}") + total=$(git rev-list --count origin/main.."${b}") + echo " ${b}: +${inc} commit(s), ${total} total vs main" +done diff --git a/scripts/py315-stack/sync-pr-numbers.sh b/scripts/py315-stack/sync-pr-numbers.sh new file mode 100755 index 00000000000..447bd5f965e --- /dev/null +++ b/scripts/py315-stack/sync-pr-numbers.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Query GitHub for PR numbers by head branch and refresh pr-numbers.env + bodies. +# Preserves manually set PRn= values; only fills empty slots from GitHub. +# +# Usage: +# ./sync-pr-numbers.sh # sync empty slots from GitHub +# ./sync-pr-numbers.sh PR6=19250 # set PR6 then sync + regenerate +set -euo pipefail + +DIR="$(cd "$(dirname "$0")" && pwd)" +ENV_FILE="$DIR/pr-numbers.env" + +declare -A EXISTING=() +if [[ -f "$ENV_FILE" ]]; then + while IFS='=' read -r key value; do + [[ "$key" =~ ^PR[0-9]+$ ]] || continue + EXISTING["$key"]="$value" + done < <(grep -E '^PR[0-9]+=' "$ENV_FILE" || true) +fi + +for arg in "$@"; do + if [[ "$arg" =~ ^(PR[0-9]+)=(.*)$ ]]; then + EXISTING["${BASH_REMATCH[1]}"]="${BASH_REMATCH[2]}" + else + echo "Usage: $0 [PRn=number ...]" >&2 + exit 1 + fi +done + +declare -a BRANCHES=( + "PR1|gab/315-monitoring-multiplexer" + "PR2|chore/315-wrapping-context" + "PR3|vlad/315-ci-matrix" + "PR4|vlad/315-ci-autoregen-lockfiles" + "PR5|vlad/315-official-support" + "PR6|vlad/315-peripheral-compat" + "PR7|vlad/profiling-native-test-install-subdir" + "PR8|vlad/ddtracepy-315-profiling-native" + "PR9|vlad/ddtracepy-315-profiling-collectors" + "PR10|vlad/ddtracepy-315-profiling-only" + "PR11|vlad/ddtracepy-315-profiling-asyncio-monitoring" + "PR12|vlad/315-profiling-dev-tooling" + "PR13|vlad/315-lib-injection-ssi" + "PR14|vlad/315-profiling-release-note" +) + +lookup_pr() { + local branch=$1 + gh pr list --head "$branch" --state all --json number --jq 'sort_by(.number) | last | .number // empty' 2>/dev/null || true +} + +TMP="$(mktemp)" +{ + echo "# PR numbers for stack navigation. ./sync-pr-numbers.sh [PRn=num ...] then regenerates bodies." +} > "$TMP" + +for entry in "${BRANCHES[@]}"; do + key="${entry%%|*}" + branch="${entry#*|}" + num="${EXISTING[$key]:-}" + + if [[ -z "$num" ]]; then + num="$(lookup_pr "$branch")" + if [[ -z "$num" && "$key" == "PR2" ]]; then + num="$(lookup_pr "gab/315-wrapping-context")" + fi + fi + + echo "${key}=${num}" >> "$TMP" + if [[ -n "$num" ]]; then + printf ' %s=%s (%s)\n' "$key" "$num" "$branch" + else + printf ' %s= (%s) — no PR yet\n' "$key" "$branch" + fi +done + +mv "$TMP" "$ENV_FILE" +echo "" +echo "Wrote ${ENV_FILE}" +"$DIR/generate-pr-urls.sh" diff --git a/scripts/run-profiling-tests b/scripts/run-profiling-tests new file mode 100755 index 00000000000..14903c7b048 --- /dev/null +++ b/scripts/run-profiling-tests @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# One-button profiling test runner for Python 3.15 (and any future version). +# +# Runs all profiling test suites on the target Python version: +# - verify_profiler_compatibility.py (async guards + real profiler samples) +# - profile (main profiling suite: asyncio, stack, threading, lock, etc.) +# - profile-memalloc (memory allocator variants: malloc, pymalloc, debug) +# - profile-uwsgi is SKIPPED — capped at Python 3.13 (uwsgi not yet 3.15-compatible) +# +# Usage: +# scripts/run-profiling-tests # default: 3.15.0a7 +# scripts/run-profiling-tests --python 3.14 # another Python version +# scripts/run-profiling-tests --quick # skip rebuild check + full riot suites; +# # only run compatibility script +# scripts/run-profiling-tests --check-only # run compatibility script only, no riot + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SCRIPTS_DIR="$REPO_ROOT/scripts" + +# --------------------------------------------------------------------------- +# Parse arguments +# --------------------------------------------------------------------------- +PYTHON_VERSION="3.15" +QUICK=false +CHECK_ONLY=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --python) + PYTHON_VERSION="$2" + shift 2 + ;; + --quick) + QUICK=true + shift + ;; + --check-only) + CHECK_ONLY=true + shift + ;; + *) + echo "Unknown argument: $1" >&2 + echo "Usage: $0 [--python VERSION] [--quick] [--check-only]" >&2 + exit 1 + ;; + esac +done + +# --------------------------------------------------------------------------- +# Locate the Python interpreter +# --------------------------------------------------------------------------- +find_python() { + local version="$1" + + # Try pyenv first (resolves 3.15 → 3.15.0a7 etc.) + if command -v pyenv &>/dev/null; then + local pyenv_version + pyenv_version=$(pyenv versions --bare 2>/dev/null | grep "^${version}" | sort -V | tail -1) + if [[ -n "$pyenv_version" ]]; then + echo "$HOME/.pyenv/versions/$pyenv_version/bin/python" + return + fi + fi + + # Try direct binary names + for candidate in "python${version}" "python3"; do + if command -v "$candidate" &>/dev/null; then + local ver + ver=$("$candidate" -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" 2>/dev/null) + if [[ "$ver" == "$version"* ]]; then + echo "$(command -v "$candidate")" + return + fi + fi + done + + echo "" +} + +PYTHON=$(find_python "$PYTHON_VERSION") +if [[ -z "$PYTHON" ]]; then + echo "ERROR: Python $PYTHON_VERSION not found." >&2 + echo "Install it with: pyenv install $PYTHON_VERSION" >&2 + exit 1 +fi + +ACTUAL_VER=$("$PYTHON" -c "import sys; print(sys.version)") +echo "=== Using Python: $ACTUAL_VER ===" +echo "=== Binary: $PYTHON ===" +echo "" + +# --------------------------------------------------------------------------- +# Step 1: Ensure the stack C extension is up-to-date +# --------------------------------------------------------------------------- +# The incremental build system skips rebuilding existing .so files. If you +# pull new commits that change stack.cpp (e.g. adding set_max_threads), the +# old .so stays in place and the stack collector silently fails at runtime — +# profiler.py catches the AttributeError and disables the collector without +# surfacing it as a test failure. Always verify and rebuild if stale. + +if [[ "$QUICK" == false && "$CHECK_ONLY" == false ]]; then + echo "--- Checking stack extension freshness ---" + + EXT_SUFFIX=$("$PYTHON" -c "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") + STACK_SO="$REPO_ROOT/ddtrace/internal/datadog/profiling/stack/_stack${EXT_SUFFIX}" + + NEEDS_REBUILD=false + + if [[ ! -f "$STACK_SO" ]]; then + echo " Stack extension not found — will build." + NEEDS_REBUILD=true + else + # Check for known required symbols by importing + if ! "$PYTHON" -c " +from ddtrace.internal.datadog.profiling.stack import _stack as s +missing = [fn for fn in ('set_max_threads', 'set_adaptive_sampling', 'set_target_overhead', 'set_max_sampling_period') if not hasattr(s, fn)] +if missing: + raise AttributeError('Missing: ' + ', '.join(missing)) +" 2>/dev/null; then + echo " Stack extension is stale (missing required symbols) — rebuilding." + NEEDS_REBUILD=true + else + echo " Stack extension is up-to-date. ✓" + fi + fi + + if [[ "$NEEDS_REBUILD" == true ]]; then + # Remove the stale .so so the incremental build detects it as missing + # and rebuilds only this extension (not Rust or other C extensions). + rm -f "$STACK_SO" + + echo " Rebuilding stack extension..." + cd "$REPO_ROOT" + "$PYTHON" -m pip install -e . --no-build-isolation -q 2>&1 | grep -E "Building|CMake|error|warning|_stack" | head -30 || true + + if [[ ! -f "$STACK_SO" ]]; then + echo "ERROR: Rebuild failed — $STACK_SO not found after pip install." >&2 + exit 1 + fi + echo " Rebuild complete. ✓" + echo "" + fi +fi + +# --------------------------------------------------------------------------- +# Step 2: Compatibility script (async guards + real profiler samples) +# --------------------------------------------------------------------------- +echo "--- Running verify_profiler_compatibility.py ---" +cd "$REPO_ROOT" + +if [[ "$QUICK" == true ]]; then + "$PYTHON" "$SCRIPTS_DIR/verify_profiler_compatibility.py" --quick +else + "$PYTHON" "$SCRIPTS_DIR/verify_profiler_compatibility.py" --compare +fi + +echo "" + +if [[ "$CHECK_ONLY" == true ]]; then + echo "=== --check-only: skipping riot suites ===" + exit 0 +fi + +# --------------------------------------------------------------------------- +# Step 3: Discover riot venv hashes dynamically for the target Python version +# --------------------------------------------------------------------------- +# Riot hashes change when riotfile.py changes, so discover them at runtime. +RIOT_PYTHON="$(cd "$REPO_ROOT" && python3 -m riot list -p "$PYTHON_VERSION" --hash-only "^profile\$" 2>/dev/null | tr '\n' ' ')" +RIOT_MEMALLOC="$(cd "$REPO_ROOT" && python3 -m riot list -p "$PYTHON_VERSION" --hash-only "^profile-memalloc\$" 2>/dev/null | tr '\n' ' ')" + +if [[ -z "$RIOT_PYTHON" ]]; then + echo "WARNING: No 'profile' riot venvs found for Python $PYTHON_VERSION. Skipping." >&2 +else + echo "--- Running profile suites: $RIOT_PYTHON---" + # shellcheck disable=SC2086 + VENV_ARGS="" + for hash in $RIOT_PYTHON; do + VENV_ARGS="$VENV_ARGS --venv $hash" + done + # shellcheck disable=SC2086 + "$SCRIPTS_DIR/run-tests" $VENV_ARGS + echo "" +fi + +if [[ -z "$RIOT_MEMALLOC" ]]; then + echo "WARNING: No 'profile-memalloc' riot venvs found for Python $PYTHON_VERSION. Skipping." >&2 +else + echo "--- Running profile-memalloc suites: $RIOT_MEMALLOC---" + VENV_ARGS="" + for hash in $RIOT_MEMALLOC; do + VENV_ARGS="$VENV_ARGS --venv $hash" + done + # shellcheck disable=SC2086 + "$SCRIPTS_DIR/run-tests" $VENV_ARGS + echo "" +fi + +echo "profile-uwsgi: SKIPPED (capped at Python 3.13 — uwsgi not yet 3.15-compatible)" +echo "" +echo "=== All profiling tests completed! ===" diff --git a/scripts/verify_profiler_compatibility.py b/scripts/verify_profiler_compatibility.py new file mode 100644 index 00000000000..391f473835a --- /dev/null +++ b/scripts/verify_profiler_compatibility.py @@ -0,0 +1,506 @@ +#!/usr/bin/env python +""" +Verify that the ddtrace profiler works correctly on any Python version. + +Collects real profiler samples and validates them — suitable for post-install +smoke tests and new Python version compatibility checks. Run this on a known-good +version to establish a baseline, then on a new version to compare. + +Usage: + # Test current interpreter + python scripts/verify_profiler_compatibility.py + + # Test a specific pyenv-installed version + python scripts/verify_profiler_compatibility.py --python 3.15.0a7 + + # Import/guard checks only — no C++ extensions required + python scripts/verify_profiler_compatibility.py --quick + + # Save current results as a baseline for this Python MAJOR.MINOR + python scripts/verify_profiler_compatibility.py --baseline + + # Compare current Python results against a saved baseline + python scripts/verify_profiler_compatibility.py --compare +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import shutil +import subprocess # nosec B404 +import sys +import tempfile +import time +from typing import Any + + +_REPO_ROOT = Path(__file__).parent.parent +_BASELINE_FILE = _REPO_ROOT / "scripts" / "profiles" / "compatibility_baselines.json" + +# Names used for asyncio tasks in the profiler sample collection suite. +# These must be unique strings that won't appear in any other samples. +_ASYNCIO_TASK_NAMES = ["compat-task-0", "compat-task-1", "compat-task-2"] +_PROFILER_RUN_SECONDS = 5.0 +_MIN_WALL_TIME_SAMPLES = 2 + + +# ============================================================================= +# SUBPROCESS MODE +# Spawned by the orchestrator under the target Python. Outputs JSON to stdout. +# All diagnostic output goes to stderr. +# ============================================================================= + + +def _suite_asyncio_guards() -> dict[str, Any]: + """Check that _asyncio.py import guards run without error. + + This is the "import smoke test" — it verifies that: + - _asyncio.py imports cleanly on the current Python version + - The ModuleWatchdog callback fires when asyncio is imported + - The hasattr guards for _scheduled_tasks, _GatheringFuture, _wait, etc. + don't raise on this version + - The asyncio policy hook path (set_event_loop wrapping) doesn't crash + """ + # Import _asyncio BEFORE asyncio to ensure the ModuleWatchdog callback is + # registered first. The callback fires when asyncio is subsequently imported. + # Importing asyncio triggers the ModuleWatchdog callback, which runs + # _call_init_asyncio() — exercising all the hasattr guards. + import asyncio + + import ddtrace.profiling._asyncio as _asyncio_mod + + if not _asyncio_mod.ASYNCIO_IMPORTED: + return {"passed": False, "error": "ASYNCIO_IMPORTED flag not set after asyncio import"} + + # Verify globals were replaced with real asyncio functions (not the no-op stubs) + if _asyncio_mod.current_task is not asyncio.current_task: + return { + "passed": False, + "error": "current_task not patched to asyncio.current_task — asyncio module watchdog may not have fired", + } + + # Exercise the policy hook: asyncio.set_event_loop() triggers + # stack.track_asyncio_loop() via the BaseDefaultEventLoopPolicy wrapper. + loop = asyncio.new_event_loop() + try: + asyncio.set_event_loop(loop) + finally: + loop.close() + asyncio.set_event_loop(None) + + return {"passed": True} + + +def _suite_profiler_samples(tmpdir: str) -> dict[str, Any]: + """Run the stack profiler with named asyncio tasks and validate pprof output. + + Checks: + - ddup and stack C++ extensions are available + - The profiler collects at least _MIN_WALL_TIME_SAMPLES wall-time samples + - asyncio task names appear in the profiler output + """ + import asyncio + + from ddtrace.internal.datadog.profiling import ddup + from ddtrace.internal.datadog.profiling import stack as _stack_ext + from ddtrace.profiling.collector import stack as stack_collector + + if not ddup.is_available: + return {"passed": False, "skipped": True, "reason": f"ddup unavailable: {ddup.failure_msg}"} + if not _stack_ext.is_available: + return {"passed": False, "skipped": True, "reason": f"stack unavailable: {_stack_ext.failure_msg}"} + + # Ensure the asyncio watchdog is registered before any loop is created. + # (It may already be imported, but importing it again is a no-op.) + import ddtrace.profiling._asyncio # noqa: F401 + + pprof_prefix = os.path.join(tmpdir, "compat") + output_filename = pprof_prefix + "." + str(os.getpid()) + + ddup.config( + env="test", + service="verify-profiler-compatibility", + version="0", + output_filename=pprof_prefix, + ) + ddup.start() + + async def _workload() -> None: + end = time.monotonic() + _PROFILER_RUN_SECONDS + while time.monotonic() < end: + await asyncio.sleep(0.05) + + with stack_collector.StackCollector(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete( + asyncio.gather(*[loop.create_task(_workload(), name=n) for n in _ASYNCIO_TASK_NAMES]) + ) + finally: + loop.close() + asyncio.set_event_loop(None) + + ddup.upload() + + result: dict[str, Any] = { + "passed": False, + "pprof_written": False, + "wall_time_samples": 0, + "asyncio_task_samples": 0, + "asyncio_task_names_seen": [], + } + + # Try to parse the pprof. Requires zstandard + google.protobuf. + try: + sys.path.insert(0, str(_REPO_ROOT)) + from tests.profiling.collector import pprof_utils + + profile = pprof_utils.parse_newest_profile(output_filename) + result["pprof_written"] = True + + wall_samples = pprof_utils.get_samples_with_value_type(profile, "wall-time") + task_samples = pprof_utils.get_samples_with_label_key(profile, "task name") + + result["wall_time_samples"] = len(wall_samples) + result["asyncio_task_samples"] = len(task_samples) + + # Extract the actual task name strings from the pprof string table + names_seen: set[str] = set() + for sample in task_samples: + label = pprof_utils.get_label_with_key(profile.string_table, sample, "task name") + if label is not None: + names_seen.add(profile.string_table[label.str]) + result["asyncio_task_names_seen"] = sorted(names_seen) + + expected = set(_ASYNCIO_TASK_NAMES) + if result["wall_time_samples"] < _MIN_WALL_TIME_SAMPLES: + result["error"] = ( + f"Too few wall-time samples: {result['wall_time_samples']} < {_MIN_WALL_TIME_SAMPLES}. " + "The profiler may not have started correctly." + ) + elif result["asyncio_task_samples"] == 0: + result["error"] = ( + "No 'task name' labels in any sample. " + "asyncio task tracking is not working — check stack.init_asyncio() and _asyncio.py." + ) + elif not names_seen & expected: + result["error"] = ( + f"Expected to see at least one of {sorted(expected)}, " + f"but got: {sorted(names_seen) or '(none)'}. " + "Task names are not being attributed to profiler samples." + ) + else: + result["passed"] = True + + except FileNotFoundError as exc: + result["error"] = f"No pprof file written to {output_filename}.*: {exc}" + except ImportError as exc: + # zstandard or protobuf not installed — degrade to file-existence check + import glob as _glob + + files = _glob.glob(pprof_prefix + "*.pprof") + result["pprof_written"] = bool(files) + result["passed"] = result["pprof_written"] + result["pprof_parse_skipped"] = True + result["pprof_parse_skip_reason"] = ( + f"{exc}. Install zstandard and protobuf in the test venv for full validation." + ) + + return result + + +def _run_subprocess(quick: bool) -> None: + """Entry point for subprocess mode. Outputs JSON to stdout.""" + results: dict[str, Any] = { + "python_version": sys.version, + "python_hexversion": hex(sys.hexversion), + } + + # Suite 1: asyncio guards — always run, no C++ required + try: + results["asyncio_guards"] = _suite_asyncio_guards() + except Exception as exc: + results["asyncio_guards"] = {"passed": False, "error": str(exc)} + + if not quick: + with tempfile.TemporaryDirectory(prefix="ddtrace-compat-") as tmpdir: + try: + results["profiler_samples"] = _suite_profiler_samples(tmpdir) + except Exception as exc: + results["profiler_samples"] = {"passed": False, "error": str(exc)} + + json.dump(results, sys.stdout, indent=2) + sys.stdout.write("\n") + + +# ============================================================================= +# ORCHESTRATOR MODE +# Finds the target Python, spawns a subprocess, formats the report. +# ============================================================================= + + +def _find_python(version: str | None) -> str: + """Return the path to the Python executable for the given version string. + + Accepts: + None → current interpreter + "3.15" → tries python3.15, then ~/.pyenv/versions/3.15.*/bin/python3 + "3.15.0a7" → tries ~/.pyenv/versions/3.15.0a7/bin/python3, then python3.15 + "/path/to/py" → used as-is + """ + if version is None: + return sys.executable + + # Absolute path + if os.sep in version: + if not os.path.isfile(version): + raise SystemExit(f"Python not found at: {version}") + return version + + # Try exact pyenv path first (handles pre-releases like 3.15.0a7) + pyenv_root = os.path.expanduser("~/.pyenv/versions") + pyenv_exact = os.path.join(pyenv_root, version, "bin", "python3") + if os.path.isfile(pyenv_exact): + return pyenv_exact + + # Try python3.X in PATH (handles short versions like "3.15") + short = version.split(".") + if len(short) >= 2: + short_name = f"python{short[0]}.{short[1]}" + found = shutil.which(short_name) + if found: + return found + + # Try pyenv glob for partial versions (e.g. "3.15" matches "3.15.0a7") + import glob as _glob + + matches = _glob.glob(os.path.join(pyenv_root, version + "*", "bin", "python3")) + if matches: + matches.sort() + return matches[-1] + + raise SystemExit( + f"Could not find Python {version!r}.\n" + f" Tried: {pyenv_exact}, {short_name if len(short) >= 2 else '(n/a)'}, PATH\n" + f" Install with: pyenv install {version}" + ) + + +def _load_baselines() -> dict[str, Any]: + if not _BASELINE_FILE.exists(): + return {} + with open(_BASELINE_FILE) as f: + data = json.load(f) + if not isinstance(data, dict): + return {} + return data + + +def _save_baselines(baselines: dict[str, Any]) -> None: + _BASELINE_FILE.parent.mkdir(parents=True, exist_ok=True) + with open(_BASELINE_FILE, "w") as f: + json.dump(baselines, f, indent=2) + f.write("\n") + + +def _baseline_key(python_exe: str) -> str: + """Return MAJOR.MINOR for the given Python executable (e.g. '3.15').""" + out = subprocess.check_output( # nosec B603 + [python_exe, "-c", "import sys; print('%d.%d' % sys.version_info[:2])"], + text=True, + ).strip() + return out + + +def _format_result(name: str, result: dict[str, Any], width: int = 22) -> str: + label = f" {name:<{width}}" + if result.get("skipped"): + reason = result.get("reason", "no reason given") + return f"{label}SKIP ({reason})" + if result.get("passed"): + extras = [] + if "wall_time_samples" in result: + extras.append(f"{result['wall_time_samples']} wall-time samples") + if "asyncio_task_names_seen" in result and result["asyncio_task_names_seen"]: + extras.append("tasks: " + ", ".join(result["asyncio_task_names_seen"])) + if result.get("pprof_parse_skipped"): + extras.append("pprof content not validated (missing zstandard/protobuf)") + suffix = f" ({', '.join(extras)})" if extras else "" + return f"{label}PASS{suffix}" + err = result.get("error", "unknown error") + return f"{label}FAIL\n {err}" + + +def _compare_with_baseline(results: dict[str, Any], baseline: dict[str, Any]) -> list[str]: + """Return a list of comparison failure messages (empty = all OK).""" + failures: list[str] = [] + + for suite in ("asyncio_guards", "profiler_samples"): + cur = results.get(suite, {}) + ref = baseline.get(suite, {}) + if not ref: + continue # baseline doesn't have this suite — skip + + cur_passed = cur.get("passed", False) + ref_passed = ref.get("passed", True) # assume baseline was passing + + if ref_passed and not cur_passed: + failures.append(f"{suite}: was PASS in baseline, now FAIL — {cur.get('error', '?')}") + + # Check sample count regression + if "wall_time_samples" in ref and "wall_time_samples" in cur: + if cur["wall_time_samples"] < ref.get("min_wall_time_samples", _MIN_WALL_TIME_SAMPLES): + failures.append( + f"{suite}: wall_time_samples dropped: {cur['wall_time_samples']} " + f"< baseline minimum {ref.get('min_wall_time_samples', _MIN_WALL_TIME_SAMPLES)}" + ) + + # Check task names + if "asyncio_task_names_seen" in ref: + expected = set(ref["asyncio_task_names_seen"]) + got = set(cur.get("asyncio_task_names_seen", [])) + missing = expected - got + if missing: + failures.append(f"{suite}: task names missing from samples: {sorted(missing)}") + + return failures + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Verify ddtrace profiler compatibility on any Python version.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--python", + metavar="VERSION", + help="Python version or path to test (e.g. 3.15.0a7, /usr/bin/python3). Default: current interpreter.", + ) + parser.add_argument( + "--quick", + action="store_true", + help="Run import/guard checks only. No C++ extensions required.", + ) + parser.add_argument( + "--baseline", + action="store_true", + help="Save results as the baseline for this Python MAJOR.MINOR.", + ) + parser.add_argument( + "--compare", + action="store_true", + help="Compare results against the saved baseline and fail if they regress.", + ) + # Internal flag — not for direct use + parser.add_argument("--subprocess", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--subprocess-quick", action="store_true", help=argparse.SUPPRESS) + + args = parser.parse_args() + + # --- Subprocess mode --- + if args.subprocess: + _run_subprocess(quick=args.subprocess_quick) + return + + # --- Orchestrator mode --- + python_exe = _find_python(args.python) + + version_str = ( + subprocess.check_output( # nosec B603 + [python_exe, "-c", "import sys; print(sys.version)"], + text=True, + ) + .strip() + .splitlines()[0] + ) + + print(f"\n=== Profiler compatibility: Python {version_str} ===") + if args.quick: + print(" (quick mode — import checks only)") + + cmd = [python_exe, __file__, "--subprocess"] + if args.quick: + cmd.append("--subprocess-quick") + + proc = subprocess.run(cmd, capture_output=True, text=True) # nosec B603 + + if proc.returncode != 0 and not proc.stdout.strip(): + print(f"\nSubprocess crashed (exit {proc.returncode}):") + print(proc.stderr or "(no stderr)") + raise SystemExit(1) + + try: + results: dict[str, Any] = json.loads(proc.stdout) + except json.JSONDecodeError: + print(f"\nCould not parse subprocess output:\n{proc.stdout}") + if proc.stderr: + print("stderr:", proc.stderr) + raise SystemExit(1) + + if proc.stderr.strip(): + print("\n[profiler stderr]") + for line in proc.stderr.strip().splitlines(): + print(f" {line}") + + print() + all_passed = True + for suite_name in ("asyncio_guards", "profiler_samples"): + if suite_name not in results: + continue + line = _format_result(suite_name, results[suite_name]) + print(line) + if not results[suite_name].get("passed") and not results[suite_name].get("skipped"): + all_passed = False + + print() + + if args.baseline: + if not all_passed: + print("Not saving baseline — some checks failed. Fix them first, then re-run with --baseline.") + else: + baseline_key = _baseline_key(python_exe) + baselines = _load_baselines() + # Only save the expected task names (not transient names like "") + # so the baseline comparison is deterministic across runs. + seen_names = set(results.get("profiler_samples", {}).get("asyncio_task_names_seen", [])) + stable_names = sorted(seen_names & set(_ASYNCIO_TASK_NAMES)) + + baselines[baseline_key] = { + "asyncio_guards": results.get("asyncio_guards", {}), + "profiler_samples": { + "passed": results.get("profiler_samples", {}).get("passed", False), + "min_wall_time_samples": _MIN_WALL_TIME_SAMPLES, + "asyncio_task_names_seen": stable_names or _ASYNCIO_TASK_NAMES, + }, + } + _save_baselines(baselines) + print(f"Baseline saved for Python {baseline_key} → {_BASELINE_FILE}") + + if args.compare: + baseline_key = _baseline_key(python_exe) + baselines = _load_baselines() + if baseline_key not in baselines: + print(f"No baseline for Python {baseline_key}. Run with --baseline on a known-good version first.") + else: + failures = _compare_with_baseline(results, baselines[baseline_key]) + if failures: + print("Baseline comparison FAILED:") + for f in failures: + print(f" - {f}") + all_passed = False + else: + print(f"Baseline comparison PASSED (vs Python {baseline_key} baseline).") + + if all_passed: + print("All checks passed.") + else: + print("Some checks FAILED. See above for details.") + raise SystemExit(1) + + +if __name__ == "__main__": + main() From ca67a73a14184d985cfc336e158fc3a66b8bbb99 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Thu, 23 Jul 2026 10:28:16 -0400 Subject: [PATCH 11/14] typing: annotate profiler compatibility script --- scripts/verify_profiler_compatibility.py | 115 +++++++++++++---------- 1 file changed, 63 insertions(+), 52 deletions(-) diff --git a/scripts/verify_profiler_compatibility.py b/scripts/verify_profiler_compatibility.py index 391f473835a..a819a05f27d 100644 --- a/scripts/verify_profiler_compatibility.py +++ b/scripts/verify_profiler_compatibility.py @@ -34,17 +34,25 @@ import sys import tempfile import time +from typing import TYPE_CHECKING from typing import Any +from typing import TextIO -_REPO_ROOT = Path(__file__).parent.parent -_BASELINE_FILE = _REPO_ROOT / "scripts" / "profiles" / "compatibility_baselines.json" +if TYPE_CHECKING: + import asyncio + + from tests.profiling.collector import pprof_pb2 + + +_REPO_ROOT: Path = Path(__file__).parent.parent +_BASELINE_FILE: Path = _REPO_ROOT / "scripts" / "profiles" / "compatibility_baselines.json" # Names used for asyncio tasks in the profiler sample collection suite. # These must be unique strings that won't appear in any other samples. -_ASYNCIO_TASK_NAMES = ["compat-task-0", "compat-task-1", "compat-task-2"] -_PROFILER_RUN_SECONDS = 5.0 -_MIN_WALL_TIME_SAMPLES = 2 +_ASYNCIO_TASK_NAMES: list[str] = ["compat-task-0", "compat-task-1", "compat-task-2"] +_PROFILER_RUN_SECONDS: float = 5.0 +_MIN_WALL_TIME_SAMPLES: int = 2 # ============================================================================= @@ -84,7 +92,7 @@ def _suite_asyncio_guards() -> dict[str, Any]: # Exercise the policy hook: asyncio.set_event_loop() triggers # stack.track_asyncio_loop() via the BaseDefaultEventLoopPolicy wrapper. - loop = asyncio.new_event_loop() + loop: asyncio.AbstractEventLoop = asyncio.new_event_loop() try: asyncio.set_event_loop(loop) finally: @@ -117,8 +125,8 @@ def _suite_profiler_samples(tmpdir: str) -> dict[str, Any]: # (It may already be imported, but importing it again is a no-op.) import ddtrace.profiling._asyncio # noqa: F401 - pprof_prefix = os.path.join(tmpdir, "compat") - output_filename = pprof_prefix + "." + str(os.getpid()) + pprof_prefix: str = os.path.join(tmpdir, "compat") + output_filename: str = pprof_prefix + "." + str(os.getpid()) ddup.config( env="test", @@ -129,12 +137,12 @@ def _suite_profiler_samples(tmpdir: str) -> dict[str, Any]: ddup.start() async def _workload() -> None: - end = time.monotonic() + _PROFILER_RUN_SECONDS + end: float = time.monotonic() + _PROFILER_RUN_SECONDS while time.monotonic() < end: await asyncio.sleep(0.05) with stack_collector.StackCollector(): - loop = asyncio.new_event_loop() + loop: asyncio.AbstractEventLoop = asyncio.new_event_loop() asyncio.set_event_loop(loop) try: loop.run_until_complete( @@ -159,11 +167,11 @@ async def _workload() -> None: sys.path.insert(0, str(_REPO_ROOT)) from tests.profiling.collector import pprof_utils - profile = pprof_utils.parse_newest_profile(output_filename) + profile: pprof_pb2.Profile = pprof_utils.parse_newest_profile(output_filename) result["pprof_written"] = True - wall_samples = pprof_utils.get_samples_with_value_type(profile, "wall-time") - task_samples = pprof_utils.get_samples_with_label_key(profile, "task name") + wall_samples: list[pprof_pb2.Sample] = pprof_utils.get_samples_with_value_type(profile, "wall-time") + task_samples: list[pprof_pb2.Sample] = pprof_utils.get_samples_with_label_key(profile, "task name") result["wall_time_samples"] = len(wall_samples) result["asyncio_task_samples"] = len(task_samples) @@ -171,12 +179,12 @@ async def _workload() -> None: # Extract the actual task name strings from the pprof string table names_seen: set[str] = set() for sample in task_samples: - label = pprof_utils.get_label_with_key(profile.string_table, sample, "task name") + label: pprof_pb2.Label | None = pprof_utils.get_label_with_key(profile.string_table, sample, "task name") if label is not None: names_seen.add(profile.string_table[label.str]) result["asyncio_task_names_seen"] = sorted(names_seen) - expected = set(_ASYNCIO_TASK_NAMES) + expected: set[str] = set(_ASYNCIO_TASK_NAMES) if result["wall_time_samples"] < _MIN_WALL_TIME_SAMPLES: result["error"] = ( f"Too few wall-time samples: {result['wall_time_samples']} < {_MIN_WALL_TIME_SAMPLES}. " @@ -202,7 +210,7 @@ async def _workload() -> None: # zstandard or protobuf not installed — degrade to file-existence check import glob as _glob - files = _glob.glob(pprof_prefix + "*.pprof") + files: list[str] = _glob.glob(pprof_prefix + "*.pprof") result["pprof_written"] = bool(files) result["passed"] = result["pprof_written"] result["pprof_parse_skipped"] = True @@ -262,23 +270,24 @@ def _find_python(version: str | None) -> str: return version # Try exact pyenv path first (handles pre-releases like 3.15.0a7) - pyenv_root = os.path.expanduser("~/.pyenv/versions") - pyenv_exact = os.path.join(pyenv_root, version, "bin", "python3") + pyenv_root: str = os.path.expanduser("~/.pyenv/versions") + pyenv_exact: str = os.path.join(pyenv_root, version, "bin", "python3") if os.path.isfile(pyenv_exact): return pyenv_exact # Try python3.X in PATH (handles short versions like "3.15") - short = version.split(".") + short: list[str] = version.split(".") + short_name: str = "" if len(short) >= 2: short_name = f"python{short[0]}.{short[1]}" - found = shutil.which(short_name) + found: str | None = shutil.which(short_name) if found: return found # Try pyenv glob for partial versions (e.g. "3.15" matches "3.15.0a7") import glob as _glob - matches = _glob.glob(os.path.join(pyenv_root, version + "*", "bin", "python3")) + matches: list[str] = _glob.glob(os.path.join(pyenv_root, version + "*", "bin", "python3")) if matches: matches.sort() return matches[-1] @@ -293,8 +302,9 @@ def _find_python(version: str | None) -> str: def _load_baselines() -> dict[str, Any]: if not _BASELINE_FILE.exists(): return {} + f: TextIO with open(_BASELINE_FILE) as f: - data = json.load(f) + data: object = json.load(f) if not isinstance(data, dict): return {} return data @@ -302,6 +312,7 @@ def _load_baselines() -> dict[str, Any]: def _save_baselines(baselines: dict[str, Any]) -> None: _BASELINE_FILE.parent.mkdir(parents=True, exist_ok=True) + f: TextIO with open(_BASELINE_FILE, "w") as f: json.dump(baselines, f, indent=2) f.write("\n") @@ -309,7 +320,7 @@ def _save_baselines(baselines: dict[str, Any]) -> None: def _baseline_key(python_exe: str) -> str: """Return MAJOR.MINOR for the given Python executable (e.g. '3.15').""" - out = subprocess.check_output( # nosec B603 + out: str = subprocess.check_output( # nosec B603 [python_exe, "-c", "import sys; print('%d.%d' % sys.version_info[:2])"], text=True, ).strip() @@ -317,21 +328,21 @@ def _baseline_key(python_exe: str) -> str: def _format_result(name: str, result: dict[str, Any], width: int = 22) -> str: - label = f" {name:<{width}}" + label: str = f" {name:<{width}}" if result.get("skipped"): - reason = result.get("reason", "no reason given") + reason: str = result.get("reason", "no reason given") return f"{label}SKIP ({reason})" if result.get("passed"): - extras = [] + extras: list[str] = [] if "wall_time_samples" in result: extras.append(f"{result['wall_time_samples']} wall-time samples") if "asyncio_task_names_seen" in result and result["asyncio_task_names_seen"]: extras.append("tasks: " + ", ".join(result["asyncio_task_names_seen"])) if result.get("pprof_parse_skipped"): extras.append("pprof content not validated (missing zstandard/protobuf)") - suffix = f" ({', '.join(extras)})" if extras else "" + suffix: str = f" ({', '.join(extras)})" if extras else "" return f"{label}PASS{suffix}" - err = result.get("error", "unknown error") + err: str = result.get("error", "unknown error") return f"{label}FAIL\n {err}" @@ -340,13 +351,13 @@ def _compare_with_baseline(results: dict[str, Any], baseline: dict[str, Any]) -> failures: list[str] = [] for suite in ("asyncio_guards", "profiler_samples"): - cur = results.get(suite, {}) - ref = baseline.get(suite, {}) + cur: dict[str, Any] = results.get(suite, {}) + ref: dict[str, Any] = baseline.get(suite, {}) if not ref: continue # baseline doesn't have this suite — skip - cur_passed = cur.get("passed", False) - ref_passed = ref.get("passed", True) # assume baseline was passing + cur_passed: bool = cur.get("passed", False) + ref_passed: bool = ref.get("passed", True) # assume baseline was passing if ref_passed and not cur_passed: failures.append(f"{suite}: was PASS in baseline, now FAIL — {cur.get('error', '?')}") @@ -361,9 +372,9 @@ def _compare_with_baseline(results: dict[str, Any], baseline: dict[str, Any]) -> # Check task names if "asyncio_task_names_seen" in ref: - expected = set(ref["asyncio_task_names_seen"]) - got = set(cur.get("asyncio_task_names_seen", [])) - missing = expected - got + expected: set[str] = set(ref["asyncio_task_names_seen"]) + got: set[str] = set(cur.get("asyncio_task_names_seen", [])) + missing: set[str] = expected - got if missing: failures.append(f"{suite}: task names missing from samples: {sorted(missing)}") @@ -371,7 +382,7 @@ def _compare_with_baseline(results: dict[str, Any], baseline: dict[str, Any]) -> def main() -> None: - parser = argparse.ArgumentParser( + parser: argparse.ArgumentParser = argparse.ArgumentParser( description="Verify ddtrace profiler compatibility on any Python version.", formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -399,7 +410,7 @@ def main() -> None: parser.add_argument("--subprocess", action="store_true", help=argparse.SUPPRESS) parser.add_argument("--subprocess-quick", action="store_true", help=argparse.SUPPRESS) - args = parser.parse_args() + args: argparse.Namespace = parser.parse_args() # --- Subprocess mode --- if args.subprocess: @@ -407,9 +418,9 @@ def main() -> None: return # --- Orchestrator mode --- - python_exe = _find_python(args.python) + python_exe: str = _find_python(args.python) - version_str = ( + version_str: str = ( subprocess.check_output( # nosec B603 [python_exe, "-c", "import sys; print(sys.version)"], text=True, @@ -422,11 +433,11 @@ def main() -> None: if args.quick: print(" (quick mode — import checks only)") - cmd = [python_exe, __file__, "--subprocess"] + cmd: list[str] = [python_exe, __file__, "--subprocess"] if args.quick: cmd.append("--subprocess-quick") - proc = subprocess.run(cmd, capture_output=True, text=True) # nosec B603 + proc: subprocess.CompletedProcess[str] = subprocess.run(cmd, capture_output=True, text=True) # nosec B603 if proc.returncode != 0 and not proc.stdout.strip(): print(f"\nSubprocess crashed (exit {proc.returncode}):") @@ -443,15 +454,15 @@ def main() -> None: if proc.stderr.strip(): print("\n[profiler stderr]") - for line in proc.stderr.strip().splitlines(): - print(f" {line}") + for stderr_line in proc.stderr.strip().splitlines(): + print(f" {stderr_line}") print() - all_passed = True + all_passed: bool = True for suite_name in ("asyncio_guards", "profiler_samples"): if suite_name not in results: continue - line = _format_result(suite_name, results[suite_name]) + line: str = _format_result(suite_name, results[suite_name]) print(line) if not results[suite_name].get("passed") and not results[suite_name].get("skipped"): all_passed = False @@ -462,12 +473,12 @@ def main() -> None: if not all_passed: print("Not saving baseline — some checks failed. Fix them first, then re-run with --baseline.") else: - baseline_key = _baseline_key(python_exe) - baselines = _load_baselines() + baseline_key: str = _baseline_key(python_exe) + baselines: dict[str, Any] = _load_baselines() # Only save the expected task names (not transient names like "") # so the baseline comparison is deterministic across runs. - seen_names = set(results.get("profiler_samples", {}).get("asyncio_task_names_seen", [])) - stable_names = sorted(seen_names & set(_ASYNCIO_TASK_NAMES)) + seen_names: set[str] = set(results.get("profiler_samples", {}).get("asyncio_task_names_seen", [])) + stable_names: list[str] = sorted(seen_names & set(_ASYNCIO_TASK_NAMES)) baselines[baseline_key] = { "asyncio_guards": results.get("asyncio_guards", {}), @@ -486,11 +497,11 @@ def main() -> None: if baseline_key not in baselines: print(f"No baseline for Python {baseline_key}. Run with --baseline on a known-good version first.") else: - failures = _compare_with_baseline(results, baselines[baseline_key]) + failures: list[str] = _compare_with_baseline(results, baselines[baseline_key]) if failures: print("Baseline comparison FAILED:") - for f in failures: - print(f" - {f}") + for failure in failures: + print(f" - {failure}") all_passed = False else: print(f"Baseline comparison PASSED (vs Python {baseline_key} baseline).") From 4d9682470fd7c9bf682e0c245f5e024fb16dc516 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Thu, 23 Jul 2026 10:31:29 -0400 Subject: [PATCH 12/14] typing: add type-annotate skill --- .claude/skills/type-annotate/SKILL.md | 86 +++++++++++++++++++++++++++ AGENTS.md | 1 + 2 files changed, 87 insertions(+) create mode 100644 .claude/skills/type-annotate/SKILL.md diff --git a/.claude/skills/type-annotate/SKILL.md b/.claude/skills/type-annotate/SKILL.md new file mode 100644 index 00000000000..00a433a8e64 --- /dev/null +++ b/.claude/skills/type-annotate/SKILL.md @@ -0,0 +1,86 @@ +--- +name: type-annotate +description: > + Add complete type annotations to new or modified Python code in dd-trace-py. + Use when writing new Python modules, when the user asks for type hints/annotations, + or before committing production or test Python changes. Covers function signatures, + locals, globals, class attributes, and validation via the lint skill. +allowed-tools: + - Bash + - Read + - Grep + - Glob + - Edit +--- + +# Type-annotate Python (dd-trace-py) + +## When to use + +- New Python files or functions in PRs +- User asks to "type-annotate", "add hints", or "fix typing" +- After implementing a feature, before commit + +## Standards (match existing ddtrace code) + +1. **Import style** — separate `typing` imports per symbol (see `ddtrace/internal/excepthook.py`), not `from typing import *`. +2. **`from __future__ import annotations`** — use only when needed for forward refs; prefer quoted strings or `TYPE_CHECKING` blocks. +3. **Annotate everything new** in production code: + - Function/method parameters and return types + - Module-level globals (`_tool_id: Optional[int] = None`) + - Class attributes when not inferred from `__init__` + - Nested functions and closures used in hot paths when practical +4. **Tests** — annotate fixtures (`-> Iterator[...]`), helper classes, and callbacks; `pytest` fixtures return typed generators. +5. **Prefer precise types** — `CodeType`, `Callable[[...], T]`, `Optional[T]`, `dict[str, Any]`; use `object` over `Any` when any object is accepted. +6. **Version gates** — keep `if sys.version_info >= (3, 15):` blocks typed inside the branch. + +## Workflow + +```bash +# 1. See what changed in the PR slice +git diff .. -- '*.py' + +# 2. Edit files — add annotations to every new/changed def, global, class attr + +# 3. Format + type-check only touched files (never raw mypy/ruff) +scripts/lint fmt -- path/to/file.py +scripts/lint typing -- path/to/file.py + +# 4. Commit on the PR branch (one-liner message) +git commit -m "typing: annotate " +git push origin +``` + +For stacked PRs, work **bottom-up**: annotate PR N, push, rebase PR N+1 onto N, repeat. + +## Common patterns + +```python +# Module global +_active: dict[str, int] = {} + +# Callback +def _on_event(code: CodeType, offset: int) -> Optional[object]: + ... + +# Fixture +@pytest.fixture() +def registered() -> Iterator[Callable[[CodeType, MonitoringEventHandler], None]]: + ... + +# TYPE_CHECKING for import cycles +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from ddtrace.profiling.collector import Collector +``` + +## Do not + +- Run `mypy` or `ruff` directly — use `scripts/lint` +- Add `# type: ignore` unless mypy requires it and a comment explains why +- Change runtime behavior while adding types + +## Related + +- Format/validate: `.claude/skills/lint/SKILL.md` +- Test changes: `.claude/skills/run-tests/SKILL.md` diff --git a/AGENTS.md b/AGENTS.md index c9944b12666..eee051176fc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,6 +111,7 @@ Use the Skill tool to invoke these. **Always prefer skills over raw commands.** |-------|---------| | `run-tests` | Running any tests or validating code changes. **Never run pytest directly.** | | `lint` | Formatting, style/type/security checks, or before committing. **Never skip before commits.** | +| `type-annotate` | Adding type hints to new or modified Python code; run before commit with `lint typing`. | | `releasenote` | Creating or updating release notes for the current branch. | | `find-cpython-usage` | Investigating CPython API dependencies or adding a new Python version. | | `compare-cpython-versions` | Comparing CPython source between two Python versions. | From b906b1e88e74d122f07ba733b17351175d85c969 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Thu, 23 Jul 2026 21:53:18 -0400 Subject: [PATCH 13/14] =?UTF-8?q?chore(py315):=20reorder=20stack=20docs=20?= =?UTF-8?q?=E2=80=94=20official=20support=20is=20PR=2014?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update pr-numbers, generate-pr-urls, MANUAL_EXISTING_PRS, and rebuild-stack to reflect linear stack with peripheral/profiling before packaging (#19254). Replacement PRs #19267–#19275 supersede wrongly-merged #19255–#19259. --- scripts/py315-stack/DRAFT_PRS.md | 122 ++++++++++-------- scripts/py315-stack/MANUAL_EXISTING_PRS.md | 49 +++---- scripts/py315-stack/generate-pr-urls.sh | 20 +-- scripts/py315-stack/pr-bodies/04-compare.url | 2 +- .../04-vlad-315-ci-autoregen-lockfiles.md | 2 +- scripts/py315-stack/pr-bodies/05-compare.url | 2 +- .../05-vlad-315-peripheral-compat.md | 14 ++ scripts/py315-stack/pr-bodies/06-compare.url | 2 +- ...ad-profiling-native-test-install-subdir.md | 14 ++ scripts/py315-stack/pr-bodies/07-compare.url | 2 +- .../07-vlad-ddtracepy-315-profiling-native.md | 14 ++ scripts/py315-stack/pr-bodies/08-compare.url | 2 +- ...vlad-ddtracepy-315-profiling-collectors.md | 14 ++ scripts/py315-stack/pr-bodies/09-compare.url | 2 +- .../09-vlad-ddtracepy-315-profiling-only.md | 14 ++ scripts/py315-stack/pr-bodies/10-compare.url | 2 +- ...racepy-315-profiling-asyncio-monitoring.md | 14 ++ scripts/py315-stack/pr-bodies/11-compare.url | 2 +- .../11-vlad-315-profiling-dev-tooling.md | 14 ++ scripts/py315-stack/pr-bodies/12-compare.url | 2 +- .../12-vlad-315-lib-injection-ssi.md | 14 ++ scripts/py315-stack/pr-bodies/13-compare.url | 2 +- .../13-vlad-315-profiling-release-note.md | 13 ++ scripts/py315-stack/pr-bodies/14-compare.url | 2 +- .../pr-bodies/14-vlad-315-official-support.md | 12 ++ scripts/py315-stack/pr-numbers.env | 20 +-- scripts/py315-stack/rebuild-stack.sh | 18 +-- 27 files changed, 268 insertions(+), 122 deletions(-) create mode 100644 scripts/py315-stack/pr-bodies/05-vlad-315-peripheral-compat.md create mode 100644 scripts/py315-stack/pr-bodies/06-vlad-profiling-native-test-install-subdir.md create mode 100644 scripts/py315-stack/pr-bodies/07-vlad-ddtracepy-315-profiling-native.md create mode 100644 scripts/py315-stack/pr-bodies/08-vlad-ddtracepy-315-profiling-collectors.md create mode 100644 scripts/py315-stack/pr-bodies/09-vlad-ddtracepy-315-profiling-only.md create mode 100644 scripts/py315-stack/pr-bodies/10-vlad-ddtracepy-315-profiling-asyncio-monitoring.md create mode 100644 scripts/py315-stack/pr-bodies/11-vlad-315-profiling-dev-tooling.md create mode 100644 scripts/py315-stack/pr-bodies/12-vlad-315-lib-injection-ssi.md create mode 100644 scripts/py315-stack/pr-bodies/13-vlad-315-profiling-release-note.md create mode 100644 scripts/py315-stack/pr-bodies/14-vlad-315-official-support.md diff --git a/scripts/py315-stack/DRAFT_PRS.md b/scripts/py315-stack/DRAFT_PRS.md index d98ff4e143d..4b3b6a5a7bf 100644 --- a/scripts/py315-stack/DRAFT_PRS.md +++ b/scripts/py315-stack/DRAFT_PRS.md @@ -78,11 +78,11 @@ Riotfile 3.15, docker testrunner, suite gating. **Existing PR:** [#19252](https://github.com/DataDog/dd-trace-py/pull/19252) — set base to `vlad/315-ci-matrix`, mark draft, paste body from [`pr-bodies/04-vlad-315-ci-autoregen-lockfiles.md`](pr-bodies/04-vlad-315-ci-autoregen-lockfiles.md). -**prev:** [#19253](https://github.com/DataDog/dd-trace-py/pull/19253) | **next:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) +**prev:** [#19253](https://github.com/DataDog/dd-trace-py/pull/19253) | **next:** [#19267](https://github.com/DataDog/dd-trace-py/pull/19267)
PR body (copy) -**prev:** [#19253](https://github.com/DataDog/dd-trace-py/pull/19253) | **next:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) +**prev:** [#19253](https://github.com/DataDog/dd-trace-py/pull/19253) | **next:** [#19267](https://github.com/DataDog/dd-trace-py/pull/19267) ## Summary @@ -97,20 +97,22 @@ Self-healing lockfile drift on PR branches.
-## 5/14 — `vlad/315-official-support` → `vlad/315-ci-autoregen-lockfiles` +## 5/14 — `vlad/315-peripheral-compat` → `vlad/315-ci-autoregen-lockfiles` -**Existing PR:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) — set base to `vlad/315-ci-autoregen-lockfiles`, mark draft, paste body from [`pr-bodies/05-vlad-315-official-support.md`](pr-bodies/05-vlad-315-official-support.md). +**Existing PR:** [#19267](https://github.com/DataDog/dd-trace-py/pull/19267) — set base to `vlad/315-ci-autoregen-lockfiles`, mark draft, paste body from [`pr-bodies/05-vlad-315-peripheral-compat.md`](pr-bodies/05-vlad-315-peripheral-compat.md). -**prev:** [#19252](https://github.com/DataDog/dd-trace-py/pull/19252) | **next:** [#19255](https://github.com/DataDog/dd-trace-py/pull/19255) +**prev:** [#19252](https://github.com/DataDog/dd-trace-py/pull/19252) | **next:** [#19268](https://github.com/DataDog/dd-trace-py/pull/19268)
PR body (copy) -**prev:** [#19252](https://github.com/DataDog/dd-trace-py/pull/19252) | **next:** [#19255](https://github.com/DataDog/dd-trace-py/pull/19255) +**prev:** [#19252](https://github.com/DataDog/dd-trace-py/pull/19252) | **next:** [#19268](https://github.com/DataDog/dd-trace-py/pull/19268) ## Summary -pyproject.toml, requirements.csv, riot lockfiles. +Graceful degradation + test skips outside wrapping core. + +Replaces merged **#19255** (wrong integration-branch base). ## Test plan @@ -120,21 +122,23 @@ pyproject.toml, requirements.csv, riot lockfiles.
-## 6/14 — `vlad/315-peripheral-compat` → `vlad/315-official-support` +## 6/14 — `vlad/profiling-native-test-install-subdir` → `vlad/315-peripheral-compat` -**Existing PR:** [#19255](https://github.com/DataDog/dd-trace-py/pull/19255) — set base to `vlad/315-official-support`, mark draft, paste body from [`pr-bodies/06-vlad-315-peripheral-compat.md`](pr-bodies/06-vlad-315-peripheral-compat.md). +**Existing PR:** [#19268](https://github.com/DataDog/dd-trace-py/pull/19268) — set base to `vlad/315-peripheral-compat`, mark draft, paste body from [`pr-bodies/06-vlad-profiling-native-test-install-subdir.md`](pr-bodies/06-vlad-profiling-native-test-install-subdir.md). -**prev:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) | **next:** [#19257](https://github.com/DataDog/dd-trace-py/pull/19257) +**prev:** [#19267](https://github.com/DataDog/dd-trace-py/pull/19267) | **next:** [#19269](https://github.com/DataDog/dd-trace-py/pull/19269)
PR body (copy) -**prev:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) | **next:** [#19257](https://github.com/DataDog/dd-trace-py/pull/19257) +**prev:** [#19267](https://github.com/DataDog/dd-trace-py/pull/19267) | **next:** [#19269](https://github.com/DataDog/dd-trace-py/pull/19269) ## Summary -Graceful degradation + test skips outside wrapping core. +INSTALL_SUBDIR for py3.15 native tests. +Replaces merged **#19257**. + ## Test plan - [ ] CI green on this branch @@ -143,21 +147,23 @@ Graceful degradation + test skips outside wrapping core.
-## 7/14 — `vlad/profiling-native-test-install-subdir` → `vlad/315-peripheral-compat` +## 7/14 — `vlad/ddtracepy-315-profiling-native` → `vlad/profiling-native-test-install-subdir` -**Existing PR:** [#19257](https://github.com/DataDog/dd-trace-py/pull/19257) — set base to `vlad/315-peripheral-compat`, mark draft, paste body from [`pr-bodies/07-vlad-profiling-native-test-install-subdir.md`](pr-bodies/07-vlad-profiling-native-test-install-subdir.md). +**Existing PR:** [#19269](https://github.com/DataDog/dd-trace-py/pull/19269) — set base to `vlad/profiling-native-test-install-subdir`, mark draft, paste body from [`pr-bodies/07-vlad-ddtracepy-315-profiling-native.md`](pr-bodies/07-vlad-ddtracepy-315-profiling-native.md). -**prev:** [#19255](https://github.com/DataDog/dd-trace-py/pull/19255) | **next:** [#19250](https://github.com/DataDog/dd-trace-py/pull/19250) +**prev:** [#19268](https://github.com/DataDog/dd-trace-py/pull/19268) | **next:** [#19270](https://github.com/DataDog/dd-trace-py/pull/19270)
PR body (copy) -**prev:** [#19255](https://github.com/DataDog/dd-trace-py/pull/19255) | **next:** [#19250](https://github.com/DataDog/dd-trace-py/pull/19250) +**prev:** [#19268](https://github.com/DataDog/dd-trace-py/pull/19268) | **next:** [#19270](https://github.com/DataDog/dd-trace-py/pull/19270) ## Summary -INSTALL_SUBDIR for py3.15 native tests. +Native profiling py3.15 ABI (Echion frame state, cmake). +Replaces merged **#19250**. + ## Test plan - [ ] CI green on this branch @@ -166,20 +172,22 @@ INSTALL_SUBDIR for py3.15 native tests.
-## 8/14 — `vlad/ddtracepy-315-profiling-native` → `vlad/profiling-native-test-install-subdir` +## 8/14 — `vlad/ddtracepy-315-profiling-collectors` → `vlad/ddtracepy-315-profiling-native` -**Existing PR:** [#19250](https://github.com/DataDog/dd-trace-py/pull/19250) — set base to `vlad/profiling-native-test-install-subdir`, mark draft, paste body from [`pr-bodies/08-vlad-ddtracepy-315-profiling-native.md`](pr-bodies/08-vlad-ddtracepy-315-profiling-native.md). +**Existing PR:** [#19270](https://github.com/DataDog/dd-trace-py/pull/19270) — set base to `vlad/ddtracepy-315-profiling-native`, mark draft, paste body from [`pr-bodies/08-vlad-ddtracepy-315-profiling-collectors.md`](pr-bodies/08-vlad-ddtracepy-315-profiling-collectors.md). -**prev:** [#19257](https://github.com/DataDog/dd-trace-py/pull/19257) | **next:** [#19251](https://github.com/DataDog/dd-trace-py/pull/19251) +**prev:** [#19269](https://github.com/DataDog/dd-trace-py/pull/19269) | **next:** [#19271](https://github.com/DataDog/dd-trace-py/pull/19271)
PR body (copy) -**prev:** [#19257](https://github.com/DataDog/dd-trace-py/pull/19257) | **next:** [#19251](https://github.com/DataDog/dd-trace-py/pull/19251) +**prev:** [#19269](https://github.com/DataDog/dd-trace-py/pull/19269) | **next:** [#19271](https://github.com/DataDog/dd-trace-py/pull/19271) ## Summary -Native profiling py3.15 ABI (Echion frame state, cmake). +Collector updates for 3.15. + +Replaces merged **#19251**. ## Test plan @@ -189,20 +197,22 @@ Native profiling py3.15 ABI (Echion frame state, cmake).
-## 9/14 — `vlad/ddtracepy-315-profiling-collectors` → `vlad/ddtracepy-315-profiling-native` +## 9/14 — `vlad/ddtracepy-315-profiling-only` → `vlad/ddtracepy-315-profiling-collectors` -**Existing PR:** [#19251](https://github.com/DataDog/dd-trace-py/pull/19251) — set base to `vlad/ddtracepy-315-profiling-native`, mark draft, paste body from [`pr-bodies/09-vlad-ddtracepy-315-profiling-collectors.md`](pr-bodies/09-vlad-ddtracepy-315-profiling-collectors.md). +**Existing PR:** [#19271](https://github.com/DataDog/dd-trace-py/pull/19271) — set base to `vlad/ddtracepy-315-profiling-collectors`, mark draft, paste body from [`pr-bodies/09-vlad-ddtracepy-315-profiling-only.md`](pr-bodies/09-vlad-ddtracepy-315-profiling-only.md). -**prev:** [#19250](https://github.com/DataDog/dd-trace-py/pull/19250) | **next:** [#19249](https://github.com/DataDog/dd-trace-py/pull/19249) +**prev:** [#19270](https://github.com/DataDog/dd-trace-py/pull/19270) | **next:** [#19272](https://github.com/DataDog/dd-trace-py/pull/19272)
PR body (copy) -**prev:** [#19250](https://github.com/DataDog/dd-trace-py/pull/19250) | **next:** [#19249](https://github.com/DataDog/dd-trace-py/pull/19249) +**prev:** [#19270](https://github.com/DataDog/dd-trace-py/pull/19270) | **next:** [#19272](https://github.com/DataDog/dd-trace-py/pull/19272) ## Summary -Collector updates for 3.15. +Profiling CI matrix and setup.py gating. + +Replaces merged **#19249**. ## Test plan @@ -212,21 +222,23 @@ Collector updates for 3.15.
-## 10/14 — `vlad/ddtracepy-315-profiling-only` → `vlad/ddtracepy-315-profiling-collectors` +## 10/14 — `vlad/ddtracepy-315-profiling-asyncio-monitoring` → `vlad/ddtracepy-315-profiling-only` -**Existing PR:** [#19249](https://github.com/DataDog/dd-trace-py/pull/19249) — set base to `vlad/ddtracepy-315-profiling-collectors`, mark draft, paste body from [`pr-bodies/10-vlad-ddtracepy-315-profiling-only.md`](pr-bodies/10-vlad-ddtracepy-315-profiling-only.md). +**Existing PR:** [#19272](https://github.com/DataDog/dd-trace-py/pull/19272) — set base to `vlad/ddtracepy-315-profiling-only`, mark draft, paste body from [`pr-bodies/10-vlad-ddtracepy-315-profiling-asyncio-monitoring.md`](pr-bodies/10-vlad-ddtracepy-315-profiling-asyncio-monitoring.md). -**prev:** [#19251](https://github.com/DataDog/dd-trace-py/pull/19251) | **next:** [#19256](https://github.com/DataDog/dd-trace-py/pull/19256) +**prev:** [#19271](https://github.com/DataDog/dd-trace-py/pull/19271) | **next:** [#19273](https://github.com/DataDog/dd-trace-py/pull/19273)
PR body (copy) -**prev:** [#19251](https://github.com/DataDog/dd-trace-py/pull/19251) | **next:** [#19256](https://github.com/DataDog/dd-trace-py/pull/19256) +**prev:** [#19271](https://github.com/DataDog/dd-trace-py/pull/19271) | **next:** [#19273](https://github.com/DataDog/dd-trace-py/pull/19273) ## Summary -Profiling CI matrix and setup.py gating. +Replace bytecode wrapping with `sys.monitoring` in `_asyncio.py`. +Replaces merged **#19256**. + ## Test plan - [ ] CI green on this branch @@ -235,20 +247,22 @@ Profiling CI matrix and setup.py gating.
-## 11/14 — `vlad/ddtracepy-315-profiling-asyncio-monitoring` → `vlad/ddtracepy-315-profiling-only` +## 11/14 — `vlad/315-profiling-dev-tooling` → `vlad/ddtracepy-315-profiling-asyncio-monitoring` -**Existing PR:** [#19256](https://github.com/DataDog/dd-trace-py/pull/19256) — set base to `vlad/ddtracepy-315-profiling-only`, mark draft, paste body from [`pr-bodies/11-vlad-ddtracepy-315-profiling-asyncio-monitoring.md`](pr-bodies/11-vlad-ddtracepy-315-profiling-asyncio-monitoring.md). +**Existing PR:** [#19273](https://github.com/DataDog/dd-trace-py/pull/19273) — set base to `vlad/ddtracepy-315-profiling-asyncio-monitoring`, mark draft, paste body from [`pr-bodies/11-vlad-315-profiling-dev-tooling.md`](pr-bodies/11-vlad-315-profiling-dev-tooling.md). -**prev:** [#19249](https://github.com/DataDog/dd-trace-py/pull/19249) | **next:** [#19260](https://github.com/DataDog/dd-trace-py/pull/19260) +**prev:** [#19272](https://github.com/DataDog/dd-trace-py/pull/19272) | **next:** [#19274](https://github.com/DataDog/dd-trace-py/pull/19274)
PR body (copy) -**prev:** [#19249](https://github.com/DataDog/dd-trace-py/pull/19249) | **next:** [#19260](https://github.com/DataDog/dd-trace-py/pull/19260) +**prev:** [#19272](https://github.com/DataDog/dd-trace-py/pull/19272) | **next:** [#19274](https://github.com/DataDog/dd-trace-py/pull/19274) ## Summary -Replace bytecode wrapping with `sys.monitoring` in `_asyncio.py`. +Profiling bring-up scripts, compatibility baselines, Echion migration runbook (GAP-03). + +Replaces merged **#19260**. ## Test plan @@ -258,20 +272,22 @@ Replace bytecode wrapping with `sys.monitoring` in `_asyncio.py`.
-## 12/14 — `vlad/315-profiling-dev-tooling` → `vlad/ddtracepy-315-profiling-asyncio-monitoring` +## 12/14 — `vlad/315-lib-injection-ssi` → `vlad/315-profiling-dev-tooling` -**Existing PR:** [#19260](https://github.com/DataDog/dd-trace-py/pull/19260) — set base to `vlad/ddtracepy-315-profiling-asyncio-monitoring`, mark draft, paste body from [`pr-bodies/12-vlad-315-profiling-dev-tooling.md`](pr-bodies/12-vlad-315-profiling-dev-tooling.md). +**Existing PR:** [#19274](https://github.com/DataDog/dd-trace-py/pull/19274) — set base to `vlad/315-profiling-dev-tooling`, mark draft, paste body from [`pr-bodies/12-vlad-315-lib-injection-ssi.md`](pr-bodies/12-vlad-315-lib-injection-ssi.md). -**prev:** [#19256](https://github.com/DataDog/dd-trace-py/pull/19256) | **next:** [#19258](https://github.com/DataDog/dd-trace-py/pull/19258) +**prev:** [#19273](https://github.com/DataDog/dd-trace-py/pull/19273) | **next:** [#19275](https://github.com/DataDog/dd-trace-py/pull/19275)
PR body (copy) -**prev:** [#19256](https://github.com/DataDog/dd-trace-py/pull/19256) | **next:** [#19258](https://github.com/DataDog/dd-trace-py/pull/19258) +**prev:** [#19273](https://github.com/DataDog/dd-trace-py/pull/19273) | **next:** [#19275](https://github.com/DataDog/dd-trace-py/pull/19275) ## Summary -Profiling bring-up scripts, compatibility baselines, Echion migration runbook (GAP-03). +SSI allow-list + wheel download for 3.15 auto-instrumentation (GAP-01). Closes #17813. + +Replaces merged **#19258**. ## Test plan @@ -281,20 +297,22 @@ Profiling bring-up scripts, compatibility baselines, Echion migration runbook (G
-## 13/14 — `vlad/315-lib-injection-ssi` → `vlad/315-profiling-dev-tooling` +## 13/14 — `vlad/315-profiling-release-note` → `vlad/315-lib-injection-ssi` -**Existing PR:** [#19258](https://github.com/DataDog/dd-trace-py/pull/19258) — set base to `vlad/315-profiling-dev-tooling`, mark draft, paste body from [`pr-bodies/13-vlad-315-lib-injection-ssi.md`](pr-bodies/13-vlad-315-lib-injection-ssi.md). +**Existing PR:** [#19275](https://github.com/DataDog/dd-trace-py/pull/19275) — set base to `vlad/315-lib-injection-ssi`, mark draft, paste body from [`pr-bodies/13-vlad-315-profiling-release-note.md`](pr-bodies/13-vlad-315-profiling-release-note.md). -**prev:** [#19260](https://github.com/DataDog/dd-trace-py/pull/19260) | **next:** [#19259](https://github.com/DataDog/dd-trace-py/pull/19259) +**prev:** [#19274](https://github.com/DataDog/dd-trace-py/pull/19274) | **next:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254)
PR body (copy) -**prev:** [#19260](https://github.com/DataDog/dd-trace-py/pull/19260) | **next:** [#19259](https://github.com/DataDog/dd-trace-py/pull/19259) +**prev:** [#19274](https://github.com/DataDog/dd-trace-py/pull/19274) | **next:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) ## Summary -SSI allow-list + wheel download for 3.15 auto-instrumentation (GAP-01). Closes #17813. +Customer-facing Reno fragment for profiling on 3.15 (GAP-02). + +Replaces merged **#19259**. ## Test plan @@ -303,19 +321,19 @@ SSI allow-list + wheel download for 3.15 auto-instrumentation (GAP-01). Closes #
-## 14/14 — `vlad/315-profiling-release-note` → `vlad/315-lib-injection-ssi` +## 14/14 — `vlad/315-official-support` → `vlad/315-profiling-release-note` -**Existing PR:** [#19259](https://github.com/DataDog/dd-trace-py/pull/19259) — set base to `vlad/315-lib-injection-ssi`, mark draft, paste body from [`pr-bodies/14-vlad-315-profiling-release-note.md`](pr-bodies/14-vlad-315-profiling-release-note.md). +**Existing PR:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) — set base to `vlad/315-profiling-release-note`, mark draft, paste body from [`pr-bodies/14-vlad-315-official-support.md`](pr-bodies/14-vlad-315-official-support.md). -**prev:** [#19258](https://github.com/DataDog/dd-trace-py/pull/19258) | **next:** — +**prev:** [#19275](https://github.com/DataDog/dd-trace-py/pull/19275) | **next:** —
PR body (copy) -**prev:** [#19258](https://github.com/DataDog/dd-trace-py/pull/19258) | **next:** — +**prev:** [#19275](https://github.com/DataDog/dd-trace-py/pull/19275) | **next:** — ## Summary -Customer-facing Reno fragment for profiling on 3.15 (GAP-02). +pyproject.toml, requirements.csv, riot lockfiles. **Stack tip — merges last.** ## Test plan diff --git a/scripts/py315-stack/MANUAL_EXISTING_PRS.md b/scripts/py315-stack/MANUAL_EXISTING_PRS.md index 99806d798a1..a91fe4b9fbd 100644 --- a/scripts/py315-stack/MANUAL_EXISTING_PRS.md +++ b/scripts/py315-stack/MANUAL_EXISTING_PRS.md @@ -1,42 +1,33 @@ -# Manual steps for the py3.15 stack PRs (EMU blocks `gh pr edit`) +# Manual steps for the py3.15 stack PRs -Branches are **rebased and force-pushed** (2026-07-23). Gab's commits are **unsigned** (not wrongly signed); -Vlad's commits are GPG-verified. `gh` cannot retarget bases or edit bodies from this EMU account — do the -steps below in the GitHub UI (or with a non-EMU token). +Linear stack (2026-07-24): each PR bases on the **previous branch**, not an integration branch. +Official 3.15 packaging support ([#19254](https://github.com/DataDog/dd-trace-py/pull/19254)) is **PR 14 / stack tip**. -## Close duplicate +## Active stack -| PR | Action | -|----|--------| -| [#19248](https://github.com/DataDog/dd-trace-py/pull/19248) | **Close** — duplicate of [#17849](https://github.com/DataDog/dd-trace-py/pull/17849) (`gab/315-wrapping-context` ≡ `chore/315-wrapping-context`) | - -## Retarget base branch (required for incremental diffs) - -| # | PR | Head branch | Set base to | +| # | PR | Head branch | Base branch | |---|-----|-------------|-------------| | 1 | [#19247](https://github.com/DataDog/dd-trace-py/pull/19247) | `gab/315-monitoring-multiplexer` | `main` | | 2 | [#17849](https://github.com/DataDog/dd-trace-py/pull/17849) | `chore/315-wrapping-context` | `gab/315-monitoring-multiplexer` | | 3 | [#19253](https://github.com/DataDog/dd-trace-py/pull/19253) | `vlad/315-ci-matrix` | `gab/315-wrapping-context` | | 4 | [#19252](https://github.com/DataDog/dd-trace-py/pull/19252) | `vlad/315-ci-autoregen-lockfiles` | `vlad/315-ci-matrix` | -| 5 | [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) | `vlad/315-official-support` | `vlad/315-ci-autoregen-lockfiles` | -| 6 | [#19255](https://github.com/DataDog/dd-trace-py/pull/19255) | `vlad/315-peripheral-compat` | `vlad/315-official-support` | -| 7 | [#19257](https://github.com/DataDog/dd-trace-py/pull/19257) | `vlad/profiling-native-test-install-subdir` | `vlad/315-peripheral-compat` | -| 8 | [#19250](https://github.com/DataDog/dd-trace-py/pull/19250) | `vlad/ddtracepy-315-profiling-native` | `vlad/profiling-native-test-install-subdir` | -| 9 | [#19251](https://github.com/DataDog/dd-trace-py/pull/19251) | `vlad/ddtracepy-315-profiling-collectors` | `vlad/ddtracepy-315-profiling-native` | -| 10 | [#19249](https://github.com/DataDog/dd-trace-py/pull/19249) | `vlad/ddtracepy-315-profiling-only` | `vlad/ddtracepy-315-profiling-collectors` | -| 11 | [#19256](https://github.com/DataDog/dd-trace-py/pull/19256) | `vlad/ddtracepy-315-profiling-asyncio-monitoring` | `vlad/ddtracepy-315-profiling-only` | -| 12 | [#19260](https://github.com/DataDog/dd-trace-py/pull/19260) | `vlad/315-profiling-dev-tooling` | `vlad/ddtracepy-315-profiling-asyncio-monitoring` | -| 13 | [#19258](https://github.com/DataDog/dd-trace-py/pull/19258) | `vlad/315-lib-injection-ssi` | `vlad/315-profiling-dev-tooling` | -| 14 | [#19259](https://github.com/DataDog/dd-trace-py/pull/19259) | `vlad/315-profiling-release-note` | `vlad/315-lib-injection-ssi` | - -After retargeting, each PR should show **1 commit** (PR 11 shows **3** asyncio commits) vs its parent branch. +| 5 | [#19267](https://github.com/DataDog/dd-trace-py/pull/19267) | `vlad/315-peripheral-compat` | `vlad/315-ci-autoregen-lockfiles` | +| 6 | [#19268](https://github.com/DataDog/dd-trace-py/pull/19268) | `vlad/profiling-native-test-install-subdir` | `vlad/315-peripheral-compat` | +| 7 | [#19269](https://github.com/DataDog/dd-trace-py/pull/19269) | `vlad/ddtracepy-315-profiling-native` | `vlad/profiling-native-test-install-subdir` | +| 8 | [#19270](https://github.com/DataDog/dd-trace-py/pull/19270) | `vlad/ddtracepy-315-profiling-collectors` | `vlad/ddtracepy-315-profiling-native` | +| 9 | [#19271](https://github.com/DataDog/dd-trace-py/pull/19271) | `vlad/ddtracepy-315-profiling-only` | `vlad/ddtracepy-315-profiling-collectors` | +| 10 | [#19272](https://github.com/DataDog/dd-trace-py/pull/19272) | `vlad/ddtracepy-315-profiling-asyncio-monitoring` | `vlad/ddtracepy-315-profiling-only` | +| 11 | [#19273](https://github.com/DataDog/dd-trace-py/pull/19273) | `vlad/315-profiling-dev-tooling` | `vlad/ddtracepy-315-profiling-asyncio-monitoring` | +| 12 | [#19274](https://github.com/DataDog/dd-trace-py/pull/19274) | `vlad/315-lib-injection-ssi` | `vlad/315-profiling-dev-tooling` | +| 13 | [#19275](https://github.com/DataDog/dd-trace-py/pull/19275) | `vlad/315-profiling-release-note` | `vlad/315-lib-injection-ssi` | +| 14 | [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) | `vlad/315-official-support` | `vlad/315-profiling-release-note` | -## Update PR description +Numbers are in [`pr-numbers.env`](pr-numbers.env). Re-run [`generate-pr-urls.sh`](generate-pr-urls.sh) after any number change. -Paste the body from `pr-bodies/NN-*.md` (first line: `**prev:** … | **next:** …`). Full index: [`DRAFT_PRS.md`](DRAFT_PRS.md). +## Superseded (do not use) -Numbers are in [`pr-numbers.env`](pr-numbers.env). Re-run `./generate-pr-urls.sh` after any number change. +These were merged into the old `vlad/315-official-support` integration branch with the wrong base. Replaced by #19267–#19275: -## Superseded closed PRs +#19255, #19257, #19250, #19251, #19249, #19256, #19260, #19258, #19259 -Do not reopen: #18488, #18503, #18504, #17624, #18389 — replaced by #19257–#19256 above. +Also do not reopen: #18488, #18503, #18504, #17624, #18389, #19248 (duplicate of #17849). diff --git a/scripts/py315-stack/generate-pr-urls.sh b/scripts/py315-stack/generate-pr-urls.sh index 7ad4d7b176e..5fc03612bb1 100755 --- a/scripts/py315-stack/generate-pr-urls.sh +++ b/scripts/py315-stack/generate-pr-urls.sh @@ -133,16 +133,16 @@ ENTRIES=( "2|chore/315-wrapping-context|gab/315-monitoring-multiplexer|chore(wrapping): Python 3.15 wrapping context support (split 2/14)|Wrapping context + bytecode_injection for 3.15 (Gab + await/send fix).|Retarget existing **#17849** to base \`gab/315-monitoring-multiplexer\`." "3|vlad/315-ci-matrix|gab/315-wrapping-context|ci(py3.15): add 3.15 to riot matrix with gated suites (split 3/14)|Riotfile 3.15, docker testrunner, suite gating.|·" "4|vlad/315-ci-autoregen-lockfiles|vlad/315-ci-matrix|ci: auto-commit regenerated riot lockfiles on PR branches (split 4/14)|Self-healing lockfile drift on PR branches.|·" - "5|vlad/315-official-support|vlad/315-ci-autoregen-lockfiles|chore(py3.15): declare official 3.15 support in packaging (split 5/14)|pyproject.toml, requirements.csv, riot lockfiles.|·" - "6|vlad/315-peripheral-compat|vlad/315-official-support|fix(py3.15): peripheral compat for profiling, logging, appsec (split 6/14)|Graceful degradation + test skips outside wrapping core.|·" - "7|vlad/profiling-native-test-install-subdir|vlad/315-peripheral-compat|chore(profiling): native test install subdirs (PROF-14200) (split 7/14)|INSTALL_SUBDIR for py3.15 native tests.|·" - "8|vlad/ddtracepy-315-profiling-native|vlad/profiling-native-test-install-subdir|chore(profiling): native C++/Rust py3.15 ABI support (split 8/14)|Native profiling py3.15 ABI (Echion frame state, cmake).|·" - "9|vlad/ddtracepy-315-profiling-collectors|vlad/ddtracepy-315-profiling-native|chore(profiling): update Python profiling collectors for py3.15 (split 9/14)|Collector updates for 3.15.|·" - "10|vlad/ddtracepy-315-profiling-only|vlad/ddtracepy-315-profiling-collectors|ci(profiling): wire py3.15 into build matrix and CI (split 10/14)|Profiling CI matrix and setup.py gating.|·" - "11|vlad/ddtracepy-315-profiling-asyncio-monitoring|vlad/ddtracepy-315-profiling-only|refactor(profiling): sys.monitoring asyncio path for py3.15 (split 11/14)|Replace bytecode wrapping with \`sys.monitoring\` in \`_asyncio.py\`.|·" - "12|vlad/315-profiling-dev-tooling|vlad/ddtracepy-315-profiling-asyncio-monitoring|docs(profiling): py3.15 dev tooling and CPython upgrade runbook (split 12/14)|Profiling bring-up scripts, compatibility baselines, Echion migration runbook (GAP-03).|·" - "13|vlad/315-lib-injection-ssi|vlad/315-profiling-dev-tooling|chore(py-315): enable lib-injection SSI for Python 3.15 (split 13/14)|SSI allow-list + wheel download for 3.15 auto-instrumentation (GAP-01). Closes #17813.|·" - "14|vlad/315-profiling-release-note|vlad/315-lib-injection-ssi|docs(releasenotes): profiling Python 3.15 support note (split 14/14)|Customer-facing Reno fragment for profiling on 3.15 (GAP-02).|·" + "5|vlad/315-peripheral-compat|vlad/315-ci-autoregen-lockfiles|fix(py3.15): peripheral compat for profiling, logging, appsec (split 5/14)|Graceful degradation + test skips outside wrapping core.|Replaces merged **#19255** (wrong integration-branch base)." + "6|vlad/profiling-native-test-install-subdir|vlad/315-peripheral-compat|chore(profiling): native test install subdirs (PROF-14200) (split 6/14)|INSTALL_SUBDIR for py3.15 native tests.|Replaces merged **#19257**." + "7|vlad/ddtracepy-315-profiling-native|vlad/profiling-native-test-install-subdir|chore(profiling): native C++/Rust py3.15 ABI support (split 7/14)|Native profiling py3.15 ABI (Echion frame state, cmake).|Replaces merged **#19250**." + "8|vlad/ddtracepy-315-profiling-collectors|vlad/ddtracepy-315-profiling-native|chore(profiling): update Python profiling collectors for py3.15 (split 8/14)|Collector updates for 3.15.|Replaces merged **#19251**." + "9|vlad/ddtracepy-315-profiling-only|vlad/ddtracepy-315-profiling-collectors|ci(profiling): wire py3.15 into build matrix and CI (split 9/14)|Profiling CI matrix and setup.py gating.|Replaces merged **#19249**." + "10|vlad/ddtracepy-315-profiling-asyncio-monitoring|vlad/ddtracepy-315-profiling-only|refactor(profiling): sys.monitoring asyncio path for py3.15 (split 10/14)|Replace bytecode wrapping with \`sys.monitoring\` in \`_asyncio.py\`.|Replaces merged **#19256**." + "11|vlad/315-profiling-dev-tooling|vlad/ddtracepy-315-profiling-asyncio-monitoring|docs(profiling): py3.15 dev tooling and CPython upgrade runbook (split 11/14)|Profiling bring-up scripts, compatibility baselines, Echion migration runbook (GAP-03).|Replaces merged **#19260**." + "12|vlad/315-lib-injection-ssi|vlad/315-profiling-dev-tooling|chore(py-315): enable lib-injection SSI for Python 3.15 (split 12/14)|SSI allow-list + wheel download for 3.15 auto-instrumentation (GAP-01). Closes #17813.|Replaces merged **#19258**." + "13|vlad/315-profiling-release-note|vlad/315-lib-injection-ssi|docs(releasenotes): profiling Python 3.15 support note (split 13/14)|Customer-facing Reno fragment for profiling on 3.15 (GAP-02).|Replaces merged **#19259**." + "14|vlad/315-official-support|vlad/315-profiling-release-note|chore(py3.15): declare official 3.15 support in packaging (split 14/14)|pyproject.toml, requirements.csv, riot lockfiles. **Stack tip — merges last.**|·" ) OUT_DIR="$(dirname "$0")/pr-bodies" diff --git a/scripts/py315-stack/pr-bodies/04-compare.url b/scripts/py315-stack/pr-bodies/04-compare.url index 0560d1a8130..17208965384 100644 --- a/scripts/py315-stack/pr-bodies/04-compare.url +++ b/scripts/py315-stack/pr-bodies/04-compare.url @@ -1 +1 @@ -https://github.com/DataDog/dd-trace-py/compare/vlad/315-ci-matrix...vlad/315-ci-autoregen-lockfiles?expand=1&title=ci%3A%20auto-commit%20regenerated%20riot%20lockfiles%20on%20PR%20branches%20%28split%204/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319253%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19253%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319254%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19254%29%0A%0A%23%23%20Summary%0A%0ASelf-healing%20lockfile%20drift%20on%20PR%20branches.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 +https://github.com/DataDog/dd-trace-py/compare/vlad/315-ci-matrix...vlad/315-ci-autoregen-lockfiles?expand=1&title=ci%3A%20auto-commit%20regenerated%20riot%20lockfiles%20on%20PR%20branches%20%28split%204/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319253%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19253%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319267%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19267%29%0A%0A%23%23%20Summary%0A%0ASelf-healing%20lockfile%20drift%20on%20PR%20branches.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/04-vlad-315-ci-autoregen-lockfiles.md b/scripts/py315-stack/pr-bodies/04-vlad-315-ci-autoregen-lockfiles.md index 212ee83b81b..893896bfb9f 100644 --- a/scripts/py315-stack/pr-bodies/04-vlad-315-ci-autoregen-lockfiles.md +++ b/scripts/py315-stack/pr-bodies/04-vlad-315-ci-autoregen-lockfiles.md @@ -1,4 +1,4 @@ -**prev:** [#19253](https://github.com/DataDog/dd-trace-py/pull/19253) | **next:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) +**prev:** [#19253](https://github.com/DataDog/dd-trace-py/pull/19253) | **next:** [#19267](https://github.com/DataDog/dd-trace-py/pull/19267) ## Summary diff --git a/scripts/py315-stack/pr-bodies/05-compare.url b/scripts/py315-stack/pr-bodies/05-compare.url index 8e2903e94be..12966f5c8a5 100644 --- a/scripts/py315-stack/pr-bodies/05-compare.url +++ b/scripts/py315-stack/pr-bodies/05-compare.url @@ -1 +1 @@ -https://github.com/DataDog/dd-trace-py/compare/vlad/315-ci-autoregen-lockfiles...vlad/315-official-support?expand=1&title=chore%28py3.15%29%3A%20declare%20official%203.15%20support%20in%20packaging%20%28split%205/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319252%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19252%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319255%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19255%29%0A%0A%23%23%20Summary%0A%0Apyproject.toml%2C%20requirements.csv%2C%20riot%20lockfiles.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 +https://github.com/DataDog/dd-trace-py/compare/vlad/315-ci-autoregen-lockfiles...vlad/315-peripheral-compat?expand=1&title=fix%28py3.15%29%3A%20peripheral%20compat%20for%20profiling%2C%20logging%2C%20appsec%20%28split%205/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319252%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19252%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319268%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19268%29%0A%0A%23%23%20Summary%0A%0AGraceful%20degradation%20%2B%20test%20skips%20outside%20wrapping%20core.%0A%0A%0AReplaces%20merged%20%2A%2A%2319255%2A%2A%20%28wrong%20integration-branch%20base%29.%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/05-vlad-315-peripheral-compat.md b/scripts/py315-stack/pr-bodies/05-vlad-315-peripheral-compat.md new file mode 100644 index 00000000000..0dc8bdd108d --- /dev/null +++ b/scripts/py315-stack/pr-bodies/05-vlad-315-peripheral-compat.md @@ -0,0 +1,14 @@ +**prev:** [#19252](https://github.com/DataDog/dd-trace-py/pull/19252) | **next:** [#19268](https://github.com/DataDog/dd-trace-py/pull/19268) + +## Summary + +Graceful degradation + test skips outside wrapping core. + + +Replaces merged **#19255** (wrong integration-branch base). + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/06-compare.url b/scripts/py315-stack/pr-bodies/06-compare.url index 24c7f5d796a..a76654ede4b 100644 --- a/scripts/py315-stack/pr-bodies/06-compare.url +++ b/scripts/py315-stack/pr-bodies/06-compare.url @@ -1 +1 @@ -https://github.com/DataDog/dd-trace-py/compare/vlad/315-official-support...vlad/315-peripheral-compat?expand=1&title=fix%28py3.15%29%3A%20peripheral%20compat%20for%20profiling%2C%20logging%2C%20appsec%20%28split%206/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319254%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19254%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319257%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19257%29%0A%0A%23%23%20Summary%0A%0AGraceful%20degradation%20%2B%20test%20skips%20outside%20wrapping%20core.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 +https://github.com/DataDog/dd-trace-py/compare/vlad/315-peripheral-compat...vlad/profiling-native-test-install-subdir?expand=1&title=chore%28profiling%29%3A%20native%20test%20install%20subdirs%20%28PROF-14200%29%20%28split%206/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319267%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19267%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319269%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19269%29%0A%0A%23%23%20Summary%0A%0AINSTALL_SUBDIR%20for%20py3.15%20native%20tests.%0A%0A%0AReplaces%20merged%20%2A%2A%2319257%2A%2A.%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/06-vlad-profiling-native-test-install-subdir.md b/scripts/py315-stack/pr-bodies/06-vlad-profiling-native-test-install-subdir.md new file mode 100644 index 00000000000..fa98aeb97f4 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/06-vlad-profiling-native-test-install-subdir.md @@ -0,0 +1,14 @@ +**prev:** [#19267](https://github.com/DataDog/dd-trace-py/pull/19267) | **next:** [#19269](https://github.com/DataDog/dd-trace-py/pull/19269) + +## Summary + +INSTALL_SUBDIR for py3.15 native tests. + + +Replaces merged **#19257**. + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/07-compare.url b/scripts/py315-stack/pr-bodies/07-compare.url index 89ac5080617..d397a5cb956 100644 --- a/scripts/py315-stack/pr-bodies/07-compare.url +++ b/scripts/py315-stack/pr-bodies/07-compare.url @@ -1 +1 @@ -https://github.com/DataDog/dd-trace-py/compare/vlad/315-peripheral-compat...vlad/profiling-native-test-install-subdir?expand=1&title=chore%28profiling%29%3A%20native%20test%20install%20subdirs%20%28PROF-14200%29%20%28split%207/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319255%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19255%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319250%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19250%29%0A%0A%23%23%20Summary%0A%0AINSTALL_SUBDIR%20for%20py3.15%20native%20tests.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 +https://github.com/DataDog/dd-trace-py/compare/vlad/profiling-native-test-install-subdir...vlad/ddtracepy-315-profiling-native?expand=1&title=chore%28profiling%29%3A%20native%20C%2B%2B/Rust%20py3.15%20ABI%20support%20%28split%207/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319268%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19268%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319270%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19270%29%0A%0A%23%23%20Summary%0A%0ANative%20profiling%20py3.15%20ABI%20%28Echion%20frame%20state%2C%20cmake%29.%0A%0A%0AReplaces%20merged%20%2A%2A%2319250%2A%2A.%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/07-vlad-ddtracepy-315-profiling-native.md b/scripts/py315-stack/pr-bodies/07-vlad-ddtracepy-315-profiling-native.md new file mode 100644 index 00000000000..afc2407a4ea --- /dev/null +++ b/scripts/py315-stack/pr-bodies/07-vlad-ddtracepy-315-profiling-native.md @@ -0,0 +1,14 @@ +**prev:** [#19268](https://github.com/DataDog/dd-trace-py/pull/19268) | **next:** [#19270](https://github.com/DataDog/dd-trace-py/pull/19270) + +## Summary + +Native profiling py3.15 ABI (Echion frame state, cmake). + + +Replaces merged **#19250**. + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/08-compare.url b/scripts/py315-stack/pr-bodies/08-compare.url index 71311ea47f2..ee542f35adc 100644 --- a/scripts/py315-stack/pr-bodies/08-compare.url +++ b/scripts/py315-stack/pr-bodies/08-compare.url @@ -1 +1 @@ -https://github.com/DataDog/dd-trace-py/compare/vlad/profiling-native-test-install-subdir...vlad/ddtracepy-315-profiling-native?expand=1&title=chore%28profiling%29%3A%20native%20C%2B%2B/Rust%20py3.15%20ABI%20support%20%28split%208/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319257%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19257%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319251%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19251%29%0A%0A%23%23%20Summary%0A%0ANative%20profiling%20py3.15%20ABI%20%28Echion%20frame%20state%2C%20cmake%29.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 +https://github.com/DataDog/dd-trace-py/compare/vlad/ddtracepy-315-profiling-native...vlad/ddtracepy-315-profiling-collectors?expand=1&title=chore%28profiling%29%3A%20update%20Python%20profiling%20collectors%20for%20py3.15%20%28split%208/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319269%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19269%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319271%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19271%29%0A%0A%23%23%20Summary%0A%0ACollector%20updates%20for%203.15.%0A%0A%0AReplaces%20merged%20%2A%2A%2319251%2A%2A.%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/08-vlad-ddtracepy-315-profiling-collectors.md b/scripts/py315-stack/pr-bodies/08-vlad-ddtracepy-315-profiling-collectors.md new file mode 100644 index 00000000000..1b57e687d0b --- /dev/null +++ b/scripts/py315-stack/pr-bodies/08-vlad-ddtracepy-315-profiling-collectors.md @@ -0,0 +1,14 @@ +**prev:** [#19269](https://github.com/DataDog/dd-trace-py/pull/19269) | **next:** [#19271](https://github.com/DataDog/dd-trace-py/pull/19271) + +## Summary + +Collector updates for 3.15. + + +Replaces merged **#19251**. + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/09-compare.url b/scripts/py315-stack/pr-bodies/09-compare.url index 11c123e65c2..808d0d40918 100644 --- a/scripts/py315-stack/pr-bodies/09-compare.url +++ b/scripts/py315-stack/pr-bodies/09-compare.url @@ -1 +1 @@ -https://github.com/DataDog/dd-trace-py/compare/vlad/ddtracepy-315-profiling-native...vlad/ddtracepy-315-profiling-collectors?expand=1&title=chore%28profiling%29%3A%20update%20Python%20profiling%20collectors%20for%20py3.15%20%28split%209/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319250%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19250%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319249%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19249%29%0A%0A%23%23%20Summary%0A%0ACollector%20updates%20for%203.15.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 +https://github.com/DataDog/dd-trace-py/compare/vlad/ddtracepy-315-profiling-collectors...vlad/ddtracepy-315-profiling-only?expand=1&title=ci%28profiling%29%3A%20wire%20py3.15%20into%20build%20matrix%20and%20CI%20%28split%209/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319270%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19270%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319272%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19272%29%0A%0A%23%23%20Summary%0A%0AProfiling%20CI%20matrix%20and%20setup.py%20gating.%0A%0A%0AReplaces%20merged%20%2A%2A%2319249%2A%2A.%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/09-vlad-ddtracepy-315-profiling-only.md b/scripts/py315-stack/pr-bodies/09-vlad-ddtracepy-315-profiling-only.md new file mode 100644 index 00000000000..308aa47b9c5 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/09-vlad-ddtracepy-315-profiling-only.md @@ -0,0 +1,14 @@ +**prev:** [#19270](https://github.com/DataDog/dd-trace-py/pull/19270) | **next:** [#19272](https://github.com/DataDog/dd-trace-py/pull/19272) + +## Summary + +Profiling CI matrix and setup.py gating. + + +Replaces merged **#19249**. + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/10-compare.url b/scripts/py315-stack/pr-bodies/10-compare.url index 34007c29a4d..0841b767d43 100644 --- a/scripts/py315-stack/pr-bodies/10-compare.url +++ b/scripts/py315-stack/pr-bodies/10-compare.url @@ -1 +1 @@ -https://github.com/DataDog/dd-trace-py/compare/vlad/ddtracepy-315-profiling-collectors...vlad/ddtracepy-315-profiling-only?expand=1&title=ci%28profiling%29%3A%20wire%20py3.15%20into%20build%20matrix%20and%20CI%20%28split%2010/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319251%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19251%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319256%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19256%29%0A%0A%23%23%20Summary%0A%0AProfiling%20CI%20matrix%20and%20setup.py%20gating.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 +https://github.com/DataDog/dd-trace-py/compare/vlad/ddtracepy-315-profiling-only...vlad/ddtracepy-315-profiling-asyncio-monitoring?expand=1&title=refactor%28profiling%29%3A%20sys.monitoring%20asyncio%20path%20for%20py3.15%20%28split%2010/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319271%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19271%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319273%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19273%29%0A%0A%23%23%20Summary%0A%0AReplace%20bytecode%20wrapping%20with%20%60sys.monitoring%60%20in%20%60_asyncio.py%60.%0A%0A%0AReplaces%20merged%20%2A%2A%2319256%2A%2A.%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/10-vlad-ddtracepy-315-profiling-asyncio-monitoring.md b/scripts/py315-stack/pr-bodies/10-vlad-ddtracepy-315-profiling-asyncio-monitoring.md new file mode 100644 index 00000000000..4de538c1eb6 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/10-vlad-ddtracepy-315-profiling-asyncio-monitoring.md @@ -0,0 +1,14 @@ +**prev:** [#19271](https://github.com/DataDog/dd-trace-py/pull/19271) | **next:** [#19273](https://github.com/DataDog/dd-trace-py/pull/19273) + +## Summary + +Replace bytecode wrapping with `sys.monitoring` in `_asyncio.py`. + + +Replaces merged **#19256**. + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/11-compare.url b/scripts/py315-stack/pr-bodies/11-compare.url index 889cb6c6e56..1b36a61cd8a 100644 --- a/scripts/py315-stack/pr-bodies/11-compare.url +++ b/scripts/py315-stack/pr-bodies/11-compare.url @@ -1 +1 @@ -https://github.com/DataDog/dd-trace-py/compare/vlad/ddtracepy-315-profiling-only...vlad/ddtracepy-315-profiling-asyncio-monitoring?expand=1&title=refactor%28profiling%29%3A%20sys.monitoring%20asyncio%20path%20for%20py3.15%20%28split%2011/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319249%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19249%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319260%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19260%29%0A%0A%23%23%20Summary%0A%0AReplace%20bytecode%20wrapping%20with%20%60sys.monitoring%60%20in%20%60_asyncio.py%60.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 +https://github.com/DataDog/dd-trace-py/compare/vlad/ddtracepy-315-profiling-asyncio-monitoring...vlad/315-profiling-dev-tooling?expand=1&title=docs%28profiling%29%3A%20py3.15%20dev%20tooling%20and%20CPython%20upgrade%20runbook%20%28split%2011/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319272%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19272%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319274%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19274%29%0A%0A%23%23%20Summary%0A%0AProfiling%20bring-up%20scripts%2C%20compatibility%20baselines%2C%20Echion%20migration%20runbook%20%28GAP-03%29.%0A%0A%0AReplaces%20merged%20%2A%2A%2319260%2A%2A.%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20CI%20green%20on%20this%20branch%0A-%20%5B%20%5D%20Stack%20merges%20cleanly%20into%20the%20next%20PR%27s%20base%20branch%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/11-vlad-315-profiling-dev-tooling.md b/scripts/py315-stack/pr-bodies/11-vlad-315-profiling-dev-tooling.md new file mode 100644 index 00000000000..a69eea394c7 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/11-vlad-315-profiling-dev-tooling.md @@ -0,0 +1,14 @@ +**prev:** [#19272](https://github.com/DataDog/dd-trace-py/pull/19272) | **next:** [#19274](https://github.com/DataDog/dd-trace-py/pull/19274) + +## Summary + +Profiling bring-up scripts, compatibility baselines, Echion migration runbook (GAP-03). + + +Replaces merged **#19260**. + +## Test plan + +- [ ] CI green on this branch +- [ ] Stack merges cleanly into the next PR's base branch + diff --git a/scripts/py315-stack/pr-bodies/12-compare.url b/scripts/py315-stack/pr-bodies/12-compare.url index a8240096826..11c9f8c6538 100644 --- a/scripts/py315-stack/pr-bodies/12-compare.url +++ b/scripts/py315-stack/pr-bodies/12-compare.url @@ -1 +1 @@ -https://github.com/DataDog/dd-trace-py/compare/vlad/ddtracepy-315-profiling-asyncio-monitoring...vlad/315-profiling-dev-tooling?expand=1&title=docs%28profiling%29%3A%20py3.15%20dev%20tooling%20and%20CPython%20upgrade%20runbook%20%28split%2012/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319256%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19256%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319258%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19258%29%0A%0A%23%23%20Summary%0A%0AProfiling%20bring-up%20scripts%2C%20compatibility%20baselines%2C%20Echion%20migration%20runbook%20%28GAP-03%29.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20%60scripts/run-profiling-tests%20--check-only%60%20passes%20on%203.15%20%28when%20available%29%0A-%20%5B%20%5D%20Docs%20build%20/%20link%20check%0A%0A&draft=1 +https://github.com/DataDog/dd-trace-py/compare/vlad/315-profiling-dev-tooling...vlad/315-lib-injection-ssi?expand=1&title=chore%28py-315%29%3A%20enable%20lib-injection%20SSI%20for%20Python%203.15%20%28split%2012/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319273%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19273%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319275%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19275%29%0A%0A%23%23%20Summary%0A%0ASSI%20allow-list%20%2B%20wheel%20download%20for%203.15%20auto-instrumentation%20%28GAP-01%29.%20Closes%20%2317813.%0A%0A%0AReplaces%20merged%20%2A%2A%2319258%2A%2A.%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20%60scripts/run-profiling-tests%20--check-only%60%20passes%20on%203.15%20%28when%20available%29%0A-%20%5B%20%5D%20Docs%20build%20/%20link%20check%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/12-vlad-315-lib-injection-ssi.md b/scripts/py315-stack/pr-bodies/12-vlad-315-lib-injection-ssi.md new file mode 100644 index 00000000000..58ec7f70825 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/12-vlad-315-lib-injection-ssi.md @@ -0,0 +1,14 @@ +**prev:** [#19273](https://github.com/DataDog/dd-trace-py/pull/19273) | **next:** [#19275](https://github.com/DataDog/dd-trace-py/pull/19275) + +## Summary + +SSI allow-list + wheel download for 3.15 auto-instrumentation (GAP-01). Closes #17813. + + +Replaces merged **#19258**. + +## Test plan + +- [ ] `scripts/run-profiling-tests --check-only` passes on 3.15 (when available) +- [ ] Docs build / link check + diff --git a/scripts/py315-stack/pr-bodies/13-compare.url b/scripts/py315-stack/pr-bodies/13-compare.url index 4fae50ae9cc..713f919f13e 100644 --- a/scripts/py315-stack/pr-bodies/13-compare.url +++ b/scripts/py315-stack/pr-bodies/13-compare.url @@ -1 +1 @@ -https://github.com/DataDog/dd-trace-py/compare/vlad/315-profiling-dev-tooling...vlad/315-lib-injection-ssi?expand=1&title=chore%28py-315%29%3A%20enable%20lib-injection%20SSI%20for%20Python%203.15%20%28split%2013/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319260%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19260%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319259%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19259%29%0A%0A%23%23%20Summary%0A%0ASSI%20allow-list%20%2B%20wheel%20download%20for%203.15%20auto-instrumentation%20%28GAP-01%29.%20Closes%20%2317813.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20lib-injection%20CI%20green%0A%0A&draft=1 +https://github.com/DataDog/dd-trace-py/compare/vlad/315-lib-injection-ssi...vlad/315-profiling-release-note?expand=1&title=docs%28releasenotes%29%3A%20profiling%20Python%203.15%20support%20note%20%28split%2013/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319274%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19274%29%20%7C%20%2A%2Anext%3A%2A%2A%20%5B%2319254%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19254%29%0A%0A%23%23%20Summary%0A%0ACustomer-facing%20Reno%20fragment%20for%20profiling%20on%203.15%20%28GAP-02%29.%0A%0A%0AReplaces%20merged%20%2A%2A%2319259%2A%2A.%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20lib-injection%20CI%20green%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/13-vlad-315-profiling-release-note.md b/scripts/py315-stack/pr-bodies/13-vlad-315-profiling-release-note.md new file mode 100644 index 00000000000..003eeb56e16 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/13-vlad-315-profiling-release-note.md @@ -0,0 +1,13 @@ +**prev:** [#19274](https://github.com/DataDog/dd-trace-py/pull/19274) | **next:** [#19254](https://github.com/DataDog/dd-trace-py/pull/19254) + +## Summary + +Customer-facing Reno fragment for profiling on 3.15 (GAP-02). + + +Replaces merged **#19259**. + +## Test plan + +- [ ] lib-injection CI green + diff --git a/scripts/py315-stack/pr-bodies/14-compare.url b/scripts/py315-stack/pr-bodies/14-compare.url index 0bfe2fe16bd..df15843bea8 100644 --- a/scripts/py315-stack/pr-bodies/14-compare.url +++ b/scripts/py315-stack/pr-bodies/14-compare.url @@ -1 +1 @@ -https://github.com/DataDog/dd-trace-py/compare/vlad/315-lib-injection-ssi...vlad/315-profiling-release-note?expand=1&title=docs%28releasenotes%29%3A%20profiling%20Python%203.15%20support%20note%20%28split%2014/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319258%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19258%29%20%7C%20%2A%2Anext%3A%2A%2A%20%E2%80%94%0A%0A%23%23%20Summary%0A%0ACustomer-facing%20Reno%20fragment%20for%20profiling%20on%203.15%20%28GAP-02%29.%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20%60riot%20run%20reno%60%20validates%20fragment%0A-%20%5B%20%5D%20Merge%20with%20PR%20that%20lifts%20profiling%20native%20gate%20for%203.15%0A%0A&draft=1 +https://github.com/DataDog/dd-trace-py/compare/vlad/315-profiling-release-note...vlad/315-official-support?expand=1&title=chore%28py3.15%29%3A%20declare%20official%203.15%20support%20in%20packaging%20%28split%2014/14%29&body=%2A%2Aprev%3A%2A%2A%20%5B%2319275%5D%28https%3A//github.com/DataDog/dd-trace-py/pull/19275%29%20%7C%20%2A%2Anext%3A%2A%2A%20%E2%80%94%0A%0A%23%23%20Summary%0A%0Apyproject.toml%2C%20requirements.csv%2C%20riot%20lockfiles.%20%2A%2AStack%20tip%20%E2%80%94%20merges%20last.%2A%2A%0A%0A%0A%23%23%20Test%20plan%0A%0A-%20%5B%20%5D%20%60riot%20run%20reno%60%20validates%20fragment%0A-%20%5B%20%5D%20Merge%20with%20PR%20that%20lifts%20profiling%20native%20gate%20for%203.15%0A%0A&draft=1 diff --git a/scripts/py315-stack/pr-bodies/14-vlad-315-official-support.md b/scripts/py315-stack/pr-bodies/14-vlad-315-official-support.md new file mode 100644 index 00000000000..75a3019ad05 --- /dev/null +++ b/scripts/py315-stack/pr-bodies/14-vlad-315-official-support.md @@ -0,0 +1,12 @@ +**prev:** [#19275](https://github.com/DataDog/dd-trace-py/pull/19275) | **next:** — + +## Summary + +pyproject.toml, requirements.csv, riot lockfiles. **Stack tip — merges last.** + + +## Test plan + +- [ ] `riot run reno` validates fragment +- [ ] Merge with PR that lifts profiling native gate for 3.15 + diff --git a/scripts/py315-stack/pr-numbers.env b/scripts/py315-stack/pr-numbers.env index f12ffe155b9..6f01cb72528 100644 --- a/scripts/py315-stack/pr-numbers.env +++ b/scripts/py315-stack/pr-numbers.env @@ -3,13 +3,13 @@ PR1=19247 PR2=17849 PR3=19253 PR4=19252 -PR5=19254 -PR6=19255 -PR7=19257 -PR8=19250 -PR9=19251 -PR10=19249 -PR11=19256 -PR12=19260 -PR13=19258 -PR14=19259 +PR5=19267 +PR6=19268 +PR7=19269 +PR8=19270 +PR9=19271 +PR10=19272 +PR11=19273 +PR12=19274 +PR13=19275 +PR14=19254 diff --git a/scripts/py315-stack/rebuild-stack.sh b/scripts/py315-stack/rebuild-stack.sh index 2c7c7d9ea56..c717d9a0d13 100755 --- a/scripts/py315-stack/rebuild-stack.sh +++ b/scripts/py315-stack/rebuild-stack.sh @@ -61,14 +61,10 @@ rebuild_layer vlad/315-ci-autoregen-lockfiles vlad/315-ci-matrix vlad \ "ci: auto-commit regenerated riot lockfiles on PR branches Part of the #17849 split (PR 4/14)." -rebuild_layer vlad/315-official-support vlad/315-ci-autoregen-lockfiles vlad \ - "chore(py3.15): declare official 3.15 support in packaging - -Part of the #17849 split (PR 5/14)." -rebuild_layer vlad/315-peripheral-compat vlad/315-official-support vlad \ +rebuild_layer vlad/315-peripheral-compat vlad/315-ci-autoregen-lockfiles vlad \ "fix(py3.15): peripheral compat for profiling, logging, and appsec -Part of the #17849 split (PR 6/14)." +Part of the #17849 split (PR 5/14)." rebuild_layer vlad/profiling-native-test-install-subdir vlad/315-peripheral-compat vlad \ "feat(profiling): add INSTALL_SUBDIR keyword to dd_wrapper_add_test @@ -120,16 +116,20 @@ Part of the post-stack follow-ups (PR 13/14)." rebuild_layer vlad/315-profiling-release-note vlad/315-lib-injection-ssi vlad \ "docs(releasenotes): add profiling Python 3.15 support note -Part of the post-stack follow-ups (PR 14/14)." +Part of the post-stack follow-ups (PR 13/14)." +rebuild_layer vlad/315-official-support vlad/315-profiling-release-note vlad \ + "chore(py3.15): declare official 3.15 support in packaging + +Part of the #17849 split (PR 14/14 — stack tip)." echo "" echo "Stack rebuilt. Branch tips:" for b in gab/315-monitoring-multiplexer chore/315-wrapping-context vlad/315-ci-matrix \ - vlad/315-ci-autoregen-lockfiles vlad/315-official-support vlad/315-peripheral-compat \ + vlad/315-ci-autoregen-lockfiles vlad/315-peripheral-compat \ vlad/profiling-native-test-install-subdir vlad/ddtracepy-315-profiling-native \ vlad/ddtracepy-315-profiling-collectors vlad/ddtracepy-315-profiling-only \ vlad/ddtracepy-315-profiling-asyncio-monitoring vlad/315-profiling-dev-tooling \ - vlad/315-lib-injection-ssi vlad/315-profiling-release-note; do + vlad/315-lib-injection-ssi vlad/315-profiling-release-note vlad/315-official-support; do inc=$(git rev-list --count "origin/${b}^..${b}" 2>/dev/null || git rev-list --count "${b}^..${b}") total=$(git rev-list --count origin/main.."${b}") echo " ${b}: +${inc} commit(s), ${total} total vs main" From 2fb7ac39a07a1f5675714f637f16de74742fdfd9 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Thu, 7 May 2026 18:26:02 -0400 Subject: [PATCH 14/14] chore(py-315): enable lib-injection SSI for Python 3.15 Bump sitecustomize allow-list to 3.16 (strict less-than) and add 3.15 to dl_wheels supported_versions. Supersedes #17977 (GAP-01); merge after the profiling stack lands. --- lib-injection/dl_wheels.py | 2 +- lib-injection/sources/sitecustomize.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-injection/dl_wheels.py b/lib-injection/dl_wheels.py index cdfce6881ef..66c3b7922a3 100755 --- a/lib-injection/dl_wheels.py +++ b/lib-injection/dl_wheels.py @@ -42,7 +42,7 @@ ) # Supported Python versions lists all python versions that can install at least one version of the ddtrace library. -supported_versions = ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] +supported_versions = ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] supported_arches = ["aarch64", "x86_64", "i686"] supported_platforms = ["musllinux_1_2", "manylinux2014"] supported_flavors = ["", "slim"] diff --git a/lib-injection/sources/sitecustomize.py b/lib-injection/sources/sitecustomize.py index 1cff81a3d87..8fc01fd0665 100644 --- a/lib-injection/sources/sitecustomize.py +++ b/lib-injection/sources/sitecustomize.py @@ -43,7 +43,7 @@ def parse_version(version): RUNTIMES_ALLOW_LIST = { "cpython": { "min": Version(version=(3, 9), constraint=""), - "max": Version(version=(3, 15), constraint=""), + "max": Version(version=(3, 16), constraint=""), } }