From ee942d5406308c9730784d316c54bffce39a3bbe Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 27 Jul 2026 03:59:41 +0000 Subject: [PATCH 1/3] fix(jit): conditionally apply match/fullmatch anchoring workaround Some PCRE2 builds (e.g. libpcre2-8 10.46, and also 10.39 here) ignore PCRE2_ANCHORED/PCRE2_ENDANCHORED when passed as match-time options to pcre2_jit_match(). Rather than always paying for the post-JIT ovector check and possible pcre2_match() re-run, probe the linked library once at module load and only enable the workaround when the probe detects a non-compliant JIT. - Add jit_anchor_fixup_needed() which compiles two tiny probe patterns, enables JIT, and calls pcre2_jit_match() with ANCHORED/ENDANCHORED to see whether the returned ovector respects the flags. - Cache the result in an atomic int set during PyInit_pcre_ext_c(). - Apply the ovector verification + interpreter re-run in Pattern_execute() only when the probe returns non-compliant. - Expose _jit_anchor_fixup_needed() for tests/debugging. - Add tests/test_jit_anchoring.py from the upstream fork to ensure match()/fullmatch() remain correct with and without the fix active. --- pcre_ext/pcre2.c | 107 +++++++++++++++++++++++++++++++++++- tests/test_jit_anchoring.py | 77 ++++++++++++++++++++++++++ 2 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 tests/test_jit_anchoring.py diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index 7756702..3937fb3 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -34,8 +34,11 @@ static ATOMIC_VAR(int) pcre2_version_initialized = 0; /* -1 unknown, 0 unsupported, 1 supported by the loaded PCRE2 runtime. */ static ATOMIC_VAR(int) offset_limit_support = ATOMIC_VAR_INIT(-1); #endif +/* -1 unknown, 0 compliant, 1 JIT ignores ANCHORED/ENDANCHORED match-time options. */ +static ATOMIC_VAR(int) jit_anchor_fixup_needed_state = ATOMIC_VAR_INIT(-1); static void detect_offset_limit_support(void); +static int jit_anchor_fixup_needed(void); /* * Releasing the GIL is only worthwhile when the PCRE2 call is expected to do @@ -1673,6 +1676,7 @@ Pattern_execute(PatternObject *self, PyObject *subject_obj, Py_ssize_t pos, int rc = 0; int attempt_jit = pattern_jit_get(self); + int jit_endanchor_uncertain = 0; pcre2_match_context *match_context = NULL; int match_context_from_pattern = 0; int match_context_used_offset_limit = 0; @@ -1771,10 +1775,24 @@ Pattern_execute(PatternObject *self, PyObject *subject_obj, Py_ssize_t pos, Py_DECREF(utf8_owner); raise_pcre_error("jit_match", rc, error_offset); return NULL; + } else if (jit_anchor_fixup_needed() && rc >= 0 && + (mode == EXEC_MODE_MATCH || mode == EXEC_MODE_FULLMATCH)) { + /* + * Some PCRE2 builds' pcre2_jit_match() silently ignore + * PCRE2_ANCHORED and PCRE2_ENDANCHORED as match-time options. + * Detected at module load; only apply the workaround when + * the linked library is non-compliant. + */ + PCRE2_SIZE *jit_ovector = pcre2_get_ovector_pointer(match_data); + if (jit_ovector == NULL || jit_ovector[0] != (PCRE2_SIZE)byte_start) { + rc = PCRE2_ERROR_NOMATCH; + } else if (mode == EXEC_MODE_FULLMATCH && jit_ovector[1] != offset_limit) { + jit_endanchor_uncertain = 1; + } } } - if (!pattern_jit_get(self)) { + if (!pattern_jit_get(self) || jit_endanchor_uncertain) { PCRE2_CALL_MAYBE_RELEASE_GIL(pcre2_match(self->code, (PCRE2_SPTR)buffer, exec_length, @@ -2397,6 +2415,7 @@ module_attach_match(PyObject *Py_UNUSED(module), PyObject *args) static PyObject *module_memory_allocator(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)); static PyObject *module_get_pcre2_version(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)); +static PyObject *module_jit_anchor_fixup_needed(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)); static void initialize_pcre2_version(void); @@ -2422,6 +2441,7 @@ static PyMethodDef module_methods[] = { {"get_jit_stack_limits", (PyCFunction)module_get_jit_stack_limits, METH_NOARGS, PyDoc_STR("Return the configured (start, max) JIT stack sizes." )}, {"set_jit_stack_limits", (PyCFunction)module_set_jit_stack_limits, METH_VARARGS, PyDoc_STR("Set the (start, max) sizes for newly created JIT stacks." )}, {"get_library_version", (PyCFunction)module_get_pcre2_version, METH_NOARGS, PyDoc_STR("Return the PCRE2 library version string." )}, + {"_jit_anchor_fixup_needed", (PyCFunction)module_jit_anchor_fixup_needed, METH_NOARGS, PyDoc_STR("Return 1 if the JIT anchoring workaround is active for this PCRE2 build." )}, {"get_allocator", (PyCFunction)module_memory_allocator, METH_NOARGS, PyDoc_STR("Return the name of the active heap allocator (tcmalloc/jemalloc/malloc)." )}, {"_cpu_ascii_vector_mode", (PyCFunction)module_cpu_ascii_vector_mode, METH_NOARGS, PyDoc_STR("Return the active ASCII vector width (0=scalar,1=SSE2,2=AVX2,3=AVX512)." )}, {"_debug_thread_cache_count", (PyCFunction)module_debug_thread_cache_count, METH_NOARGS, PyDoc_STR("Return the number of live thread cache states (requires PYPCRE_DEBUG=1)." )}, @@ -2495,6 +2515,84 @@ detect_offset_limit_support(void) #endif } +static int +jit_anchor_fixup_needed(void) +{ + int current = atomic_load_explicit(&jit_anchor_fixup_needed_state, memory_order_acquire); + if (current != -1) { + return current; + } + + int needed = 0; + int error_code = 0; + PCRE2_SIZE error_offset = 0; + + /* + * Probe 1: PCRE2_ANCHORED at match time must force a start-at-offset match. + * A compliant JIT run on "X2025-10-08" with pattern \\d+ should return + * PCRE2_ERROR_NOMATCH because the subject does not start with a digit. + */ + pcre2_code *code = pcre2_compile((PCRE2_SPTR)"\\d+", 3, 0, &error_code, &error_offset, NULL); + if (code != NULL) { + int jit_rc = pcre2_jit_compile(code, PCRE2_JIT_COMPLETE); + if (jit_rc >= 0) { + pcre2_match_data *match_data = pcre2_match_data_create(2, NULL); + if (match_data != NULL) { + int rc = pcre2_jit_match(code, + (PCRE2_SPTR)"X2025-10-08", + 11, + 0, + PCRE2_ANCHORED, + match_data, + NULL); + if (rc >= 0) { + PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data); + if (ovector != NULL && ovector[0] != 0) { + needed = 1; + } + } + pcre2_match_data_free(match_data); + } + } + pcre2_code_free(code); + } + + if (!needed) { + /* + * Probe 2: PCRE2_ENDANCHORED at match time must force the match to end + * at the end of the subject. Pattern "a|ab" on "ab" must match "ab", + * not just "a". A non-compliant JIT may return the shorter match. + */ + code = pcre2_compile((PCRE2_SPTR)"a|ab", 4, 0, &error_code, &error_offset, NULL); + if (code != NULL) { + int jit_rc = pcre2_jit_compile(code, PCRE2_JIT_COMPLETE); + if (jit_rc >= 0) { + pcre2_match_data *match_data = pcre2_match_data_create(2, NULL); + if (match_data != NULL) { + int rc = pcre2_jit_match(code, + (PCRE2_SPTR)"ab", + 2, + 0, + PCRE2_ANCHORED | PCRE2_ENDANCHORED, + match_data, + NULL); + if (rc >= 0) { + PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data); + if (ovector == NULL || ovector[0] != 0 || ovector[1] != 2) { + needed = 1; + } + } + pcre2_match_data_free(match_data); + } + } + pcre2_code_free(code); + } + } + + atomic_store_explicit(&jit_anchor_fixup_needed_state, needed, memory_order_release); + return needed; +} + PyMODINIT_FUNC PyInit_pcre_ext_c(void) { @@ -2557,6 +2655,7 @@ PyInit_pcre_ext_c(void) } detect_offset_limit_support(); + (void)jit_anchor_fixup_needed(); Py_INCREF(&PatternType); if (PyModule_AddObject(module, "Pattern", (PyObject *)&PatternType) < 0) { @@ -2626,6 +2725,12 @@ module_get_pcre2_version(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) return PyUnicode_FromString(pcre2_library_version); } +static PyObject * +module_jit_anchor_fixup_needed(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + return PyLong_FromLong(jit_anchor_fixup_needed()); +} + static void initialize_pcre2_version(void) { diff --git a/tests/test_jit_anchoring.py b/tests/test_jit_anchoring.py new file mode 100644 index 0000000..ac6ebb2 --- /dev/null +++ b/tests/test_jit_anchoring.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: 2025 ModelCloud.ai +# SPDX-FileCopyrightText: 2025 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +"""Regression tests for JIT anchoring correctness. + +Some PCRE2 builds' pcre2_jit_match() silently ignores PCRE2_ANCHORED and +PCRE2_ENDANCHORED when passed as match-time options, which previously let +Pattern.match()/fullmatch() report matches at the wrong position (or ones +that don't reach the required end) whenever the pattern lacked a literal +first character for the has_first_literal fast path to filter first. +""" + +import pcre_ext_c +import pytest + + +def _compile(pattern, *, jit): + return pcre_ext_c.compile(pattern, jit=jit) + + +@pytest.mark.parametrize("jit", [True, False], ids=["jit", "nojit"]) +class TestMatchAnchoring: + def test_match_rejects_non_start_occurrence(self, jit): + # No literal first character, so the has_first_literal fast path + # can't mask a broken ANCHORED option. + pattern = _compile(rb"\d{4}-\d{2}-\d{2}", jit=jit) + assert pattern.match(b"X2025-10-08") is None + + def test_match_accepts_start_occurrence(self, jit): + pattern = _compile(rb"\d{4}-\d{2}-\d{2}", jit=jit) + match = pattern.match(b"2025-10-08trailing") + assert match is not None + assert match.span() == (0, 10) + + def test_match_rejects_non_start_occurrence_str(self, jit): + pattern = _compile(r"\d{4}-\d{2}-\d{2}", jit=jit) + assert pattern.match("X2025-10-08") is None + + +@pytest.mark.parametrize("jit", [True, False], ids=["jit", "nojit"]) +class TestFullmatchAnchoring: + def test_fullmatch_exact_span(self, jit): + pattern = _compile(rb"\d{4}-\d{2}-\d{2}", jit=jit) + match = pattern.fullmatch(b"2025-10-08") + assert match is not None + assert match.span() == (0, 10) + + def test_fullmatch_rejects_trailing_content(self, jit): + pattern = _compile(rb"\d{4}-\d{2}-\d{2}", jit=jit) + assert pattern.fullmatch(b"2025-10-08X") is None + + def test_fullmatch_rejects_leading_content(self, jit): + pattern = _compile(rb"\d{4}-\d{2}-\d{2}", jit=jit) + assert pattern.fullmatch(b"X2025-10-08") is None + + def test_fullmatch_reexplores_alternation_to_reach_end(self, jit): + # A naive engine that ignores end-anchoring would greedily accept + # the leftmost alternative ("a") and stop; a correct fullmatch must + # backtrack into the "ab" alternative to cover the whole subject. + pattern = _compile(rb"a|ab", jit=jit) + match = pattern.fullmatch(b"ab") + assert match is not None + assert match.span() == (0, 2) + + def test_fullmatch_short_alternative_still_works(self, jit): + pattern = _compile(rb"a|ab", jit=jit) + match = pattern.fullmatch(b"a") + assert match is not None + assert match.span() == (0, 1) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__])) From af763995a29ea04f792e16e9fd0d7bb494e3ccd6 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 27 Jul 2026 04:06:39 +0000 Subject: [PATCH 2/3] test(jit): add fullmatch+endpos+offset-limit regression tests --- tests/test_jit_anchoring.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_jit_anchoring.py b/tests/test_jit_anchoring.py index ac6ebb2..957df46 100644 --- a/tests/test_jit_anchoring.py +++ b/tests/test_jit_anchoring.py @@ -70,6 +70,29 @@ def test_fullmatch_short_alternative_still_works(self, jit): assert match is not None assert match.span() == (0, 1) + def test_fullmatch_with_endpos_and_trailing_content(self, jit): + # endpos sets the PCRE2 offset limit; the match must end at that + # limit, not at the end of the full subject buffer. + pattern = _compile(rb"\d{4}-\d{2}-\d{2}", jit=jit) + match = pattern.fullmatch(b"2025-10-08X", endpos=10) + assert match is not None + assert match.span() == (0, 10) + + def test_fullmatch_with_endpos_rejects_overrun(self, jit): + pattern = _compile(rb"\d{4}-\d{2}-\d{2}", jit=jit) + assert pattern.fullmatch(b"2025-10-08XX", endpos=11) is None + + def test_fullmatch_endpos_respects_alternation(self, jit): + # With a trailing character after the endpos, the fallback + # pcre2_match() re-run must still anchor at endpos. + pattern = _compile(rb"a|ab", jit=jit) + match = pattern.fullmatch(b"abX", endpos=2) + assert match is not None + assert match.span() == (0, 2) + match = pattern.fullmatch(b"abX", endpos=1) + assert match is not None + assert match.span() == (0, 1) + if __name__ == "__main__": import sys From 21f563fd4f8e771095a6b6b68db7c712e138d9f8 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 27 Jul 2026 04:13:09 +0000 Subject: [PATCH 3/3] fix(jit): truncate interpreter re-run to offset_limit for fullmatch endpos fallback When the JIT anchoring workaround needs to re-run fullmatch through pcre2_match(), pass only the requested endpos prefix as the subject length. This makes PCRE2_ENDANCHORED anchor at the intended boundary regardless of whether the active PCRE2 build treats PCRE2_USE_OFFSET_LIMIT as influencing end-anchored matches. Adds endpos regression tests for match/fullmatch/search/finditer with trailing content, pos+endpos, and multibyte text. --- pcre_ext/pcre2.c | 18 ++++++++-- tests/test_jit_anchoring.py | 66 +++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index 3937fb3..f1a3ad9 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -1793,14 +1793,28 @@ Pattern_execute(PatternObject *self, PyObject *subject_obj, Py_ssize_t pos, } if (!pattern_jit_get(self) || jit_endanchor_uncertain) { + /* + * For the fullmatch JIT fallback, truncate the interpreter re-run + * to the requested endpos (offset_limit). This guarantees that + * PCRE2_ENDANCHORED anchors to the intended boundary even on PCRE2 + * builds where PCRE2_USE_OFFSET_LIMIT does not influence end + * anchoring for the interpreter re-run. + */ + PCRE2_SIZE interpreter_length = exec_length; + if (jit_endanchor_uncertain) { + interpreter_length = offset_limit; + if (interpreter_length < (PCRE2_SIZE)byte_start) { + interpreter_length = (PCRE2_SIZE)byte_start; + } + } PCRE2_CALL_MAYBE_RELEASE_GIL(pcre2_match(self->code, (PCRE2_SPTR)buffer, - exec_length, + interpreter_length, (PCRE2_SIZE)byte_start, match_options, match_data, match_context), - exec_length); + interpreter_length); } if (rc == PCRE2_ERROR_NOMATCH) { diff --git a/tests/test_jit_anchoring.py b/tests/test_jit_anchoring.py index 957df46..d5af484 100644 --- a/tests/test_jit_anchoring.py +++ b/tests/test_jit_anchoring.py @@ -94,6 +94,72 @@ def test_fullmatch_endpos_respects_alternation(self, jit): assert match.span() == (0, 1) +@pytest.mark.parametrize("jit", [True, False], ids=["jit", "nojit"]) +class TestEndposWithTrailingContent: + """endpos must be respected even when the subject buffer is longer.""" + + def test_match_endpos_trailing_content(self, jit): + pattern = _compile(rb"\d+", jit=jit) + match = pattern.match(b"12345X", endpos=3) + assert match is not None + assert match.span() == (0, 3) + + def test_fullmatch_endpos_trailing_content(self, jit): + pattern = _compile(rb"\d+", jit=jit) + match = pattern.fullmatch(b"12345X", endpos=3) + assert match is not None + assert match.span() == (0, 3) + assert pattern.fullmatch(b"12345X", endpos=6) is None + + def test_fullmatch_endpos_dot_star(self, jit): + pattern = _compile(rb".*", jit=jit) + assert pattern.fullmatch(b"abX", endpos=0).span() == (0, 0) + assert pattern.fullmatch(b"abX", endpos=1).span() == (0, 1) + assert pattern.fullmatch(b"abX", endpos=2).span() == (0, 2) + assert pattern.fullmatch(b"abX", endpos=3).span() == (0, 3) + + def test_fullmatch_endpos_alternation(self, jit): + pattern = _compile(rb"a|ab", jit=jit) + assert pattern.fullmatch(b"abX", endpos=1).span() == (0, 1) + assert pattern.fullmatch(b"abX", endpos=2).span() == (0, 2) + assert pattern.fullmatch(b"abX", endpos=3) is None + + def test_fullmatch_endpos_pos(self, jit): + pattern = _compile(rb"\d+", jit=jit) + match = pattern.fullmatch(b"X12345X", pos=1, endpos=6) + assert match is not None + assert match.span() == (1, 6) + + def test_match_endpos_pos(self, jit): + pattern = _compile(rb"\d+", jit=jit) + match = pattern.match(b"X12345X", pos=1, endpos=4) + assert match is not None + assert match.span() == (1, 4) + + def test_search_endpos_trailing_content(self, jit): + pattern = _compile(rb"\d+", jit=jit) + match = pattern.search(b"X12345X", endpos=4) + assert match is not None + assert match.span() == (1, 4) + + def test_finditer_endpos_trailing_content(self, jit): + pattern = _compile(rb"\d+", jit=jit) + matches = list(pattern.finditer(b"12X34", endpos=3)) + assert [m.group() for m in matches] == [b"12"] + + def test_fullmatch_endpos_text_multibyte(self, jit): + pattern = pcre_ext_c.compile( + r"\w+", + jit=jit, + flags=pcre_ext_c.PCRE2_UTF | pcre_ext_c.PCRE2_UCP, + ) + match = pattern.fullmatch("café!", endpos=4) + assert match is not None + assert match.span() == (0, 4) + assert match.group() == "café" + assert pattern.fullmatch("café!", endpos=5) is None + + if __name__ == "__main__": import sys