From 10e7514c62a32260079d023e4da069d8a60656e7 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 27 Jul 2026 04:21:46 +0000 Subject: [PATCH 1/3] feat: zero-copy buffer-protocol subject support (mmap, bytearray, array) Port the e08f2ab buffer support from the iforapsy fork: - Add buffer_view_from_object() helper in pcre_ext/util.c to obtain a contiguous memoryview over bytes-like objects without copying. - Wire the zero-copy path into Pattern_execute() and Pattern_create_finditer() so mmap.mmap, bytearray, array.array, and other buffer-protocol objects can be matched directly. - Broaden is_bytes_like() in pcre/re_compat.py to accept buffer-protocol exporters by attempting PyMemoryView_FromObject() (with BufferError handled). - Add UTF-8 validation for buffer-protocol subjects when the pattern has PCRE2_UTF, preventing PCRE2_NO_UTF_CHECK from passing malformed UTF-8 to PCRE2. - Add tests/test_mmap.py exercising match/fullmatch/search/finditer/split/sub against mmap and bytearray subjects, non-contiguous buffer rejection, and lifetime behavior (mmap cannot be closed while a match is alive). The GIL=0 object-lifetime contract is safe because PyMemoryView_FromObject holds the exporter alive and pins its buffer for the lifetime of the match or finditer iterator. Concurrent mutation of writable buffers remains a caller contract, not something the library can prevent; this is documented in the README section added by the original c8e8e4b commit and already exists in the repo's README. --- pcre/re_compat.py | 12 +++- pcre_ext/pcre2.c | 60 +++++++++++++++-- pcre_ext/pcre2_module.h | 1 + pcre_ext/util.c | 35 ++++++++++ tests/test_mmap.py | 140 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 240 insertions(+), 8 deletions(-) create mode 100644 tests/test_mmap.py diff --git a/pcre/re_compat.py b/pcre/re_compat.py index 2a7ece9..04af971 100644 --- a/pcre/re_compat.py +++ b/pcre/re_compat.py @@ -26,7 +26,17 @@ def prepare_subject(subject: Any) -> Any: def is_bytes_like(value: Any) -> bool: - return isinstance(value, (bytes, bytearray, memoryview)) + if isinstance(value, (bytes, bytearray, memoryview)): + return True + if isinstance(value, str): + return False + # Covers other buffer-protocol objects (e.g. mmap.mmap) without copying + # their contents; memoryview() just borrows the exporter's buffer. + try: + memoryview(value) + except TypeError: + return False + return True def normalise_count(value: Any) -> int | None: diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index f1a3ad9..dbc1d9a 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -1171,7 +1171,10 @@ create_match_object(PatternObject *pattern, match->utf8_length = utf8_length; match->public_pos = pos; match->public_endpos = endpos; - match->subject_is_bytes = PyBytes_Check(subject_obj); + /* Anything that isn't str (bytes, or a buffer-protocol object such as + mmap.mmap) is treated as raw byte data: offsets are byte offsets and + group values are returned as bytes. */ + match->subject_is_bytes = !PyUnicode_Check(subject_obj); return match; } @@ -1255,8 +1258,29 @@ Pattern_create_finditer(PatternObject *pattern, iter->subject_length_bytes = utf8_length; iter->utf8_data = utf8_data; } + } else if (PyObject_CheckBuffer(subject_obj)) { + /* Zero-copy path for e.g. mmap.mmap: get a pointer straight into the + exporter's own storage instead of copying it into a new object. */ + const char *buf_data = NULL; + Py_ssize_t buf_length = 0; + PyObject *buf_view = buffer_view_from_object(subject_obj, &buf_data, &buf_length); + if (buf_view == NULL) { + goto error; + } + iter->subject_is_bytes = 1; + iter->subject_length_bytes = buf_length; + iter->logical_length = buf_length; + iter->utf8_data = buf_data; + iter->utf8_owner = buf_view; + if (ensure_valid_utf8_for_bytes_subject(pattern, + iter->subject_is_bytes, + iter->utf8_data, + iter->subject_length_bytes) < 0) { + goto error; + } } else { - PyErr_SetString(PyExc_TypeError, "subject must be str or bytes"); + PyErr_SetString(PyExc_TypeError, + "subject must be str, bytes, or a bytes-like buffer object (e.g. mmap.mmap)"); goto error; } @@ -1362,7 +1386,7 @@ Pattern_create_finditer(PatternObject *pattern, iter->jit_stack = jit_stack; /* * The UTF-8 validity of the subject has already been established either by - * Python (str) or by ensure_valid_utf8_for_bytes_subject (bytes+UTF). + * Python (str) or by ensure_valid_utf8_for_bytes_subject (bytes/buffer+UTF). * Only skip PCRE2's UTF-8 check for the full validated range so partial * byte ranges cannot end inside a multi-byte sequence. */ @@ -1557,8 +1581,30 @@ Pattern_execute(PatternObject *self, PyObject *subject_obj, Py_ssize_t pos, buffer = utf8_data; subject_length_bytes = utf8_length; } + } else if (PyObject_CheckBuffer(subject_obj)) { + /* Zero-copy path for e.g. mmap.mmap: get a pointer straight into the + exporter's own storage instead of copying it into a new object. */ + const char *buf_data = NULL; + Py_ssize_t buf_length = 0; + PyObject *buf_view = buffer_view_from_object(subject_obj, &buf_data, &buf_length); + if (buf_view == NULL) { + return NULL; + } + utf8_owner = buf_view; + buffer = buf_data; + subject_length_bytes = buf_length; + logical_length = buf_length; + subject_is_bytes = 1; + if (ensure_valid_utf8_for_bytes_subject(self, + subject_is_bytes, + buffer, + subject_length_bytes) < 0) { + Py_DECREF(utf8_owner); + return NULL; + } } else { - PyErr_SetString(PyExc_TypeError, "expected str or bytes"); + PyErr_SetString(PyExc_TypeError, + "expected str, bytes, or a bytes-like buffer object (e.g. mmap.mmap)"); return NULL; } @@ -1657,9 +1703,9 @@ Pattern_execute(PatternObject *self, PyObject *subject_obj, Py_ssize_t pos, } /* * For text subjects we already own a UTF-8 pointer that Python validated. - * For bytes subjects with PCRE2_UTF we explicitly validated UTF-8 above. - * Only skip the PCRE2 UTF-8 check when the entire validated buffer is used; - * partial byte ranges may end inside a multi-byte sequence. + * For bytes/buffer-protocol subjects with PCRE2_UTF we explicitly validated + * UTF-8 above. Only skip the PCRE2 UTF-8 check when the entire validated + * buffer is used; partial byte ranges may end inside a multi-byte sequence. */ if (!subject_is_bytes || (byte_start == 0 && byte_end == subject_length_bytes)) { diff --git a/pcre_ext/pcre2_module.h b/pcre_ext/pcre2_module.h index b4996a1..92f0a2d 100644 --- a/pcre_ext/pcre2_module.h +++ b/pcre_ext/pcre2_module.h @@ -128,6 +128,7 @@ PyObject *module_clear_pattern_cache(PyObject *module, PyObject *args); /* Utilities */ int env_flag_is_true(const char *value); PyObject *bytes_from_text(PyObject *obj); +PyObject *buffer_view_from_object(PyObject *obj, const char **data_out, Py_ssize_t *length_out); Py_ssize_t utf8_offset_to_index(const char *data, Py_ssize_t length); int utf8_index_to_offset(PyObject *unicode_obj, Py_ssize_t index, Py_ssize_t *offset_out); PyObject *create_groupindex_dict(pcre2_code *code); diff --git a/pcre_ext/util.c b/pcre_ext/util.c index ebe8764..917b2e0 100644 --- a/pcre_ext/util.c +++ b/pcre_ext/util.c @@ -42,6 +42,41 @@ popcountll(uint64_t value) } #endif +PyObject * +buffer_view_from_object(PyObject *obj, const char **data_out, Py_ssize_t *length_out) +{ + /* + * Wrap any buffer-protocol object (e.g. mmap.mmap, bytearray) in a + * memoryview so we get a direct pointer into its existing storage. The + * memoryview pins the exporter's buffer (preventing e.g. mmap.close()) + * for as long as it is kept alive, so callers must hold a reference to + * the returned view for as long as data_out/length_out are used. + * + * PyMemoryView_FromObject is required here rather than a manual + * PyObject_GetBuffer + PyMemoryView_FromBuffer: the latter leaves the + * exporter's buffer export count permanently elevated (observed as + * mmap.close() raising BufferError even after the memoryview and all + * its referents are collected), whereas PyMemoryView_FromObject + * correctly releases the buffer when the memoryview is deallocated. + */ + PyObject *view = PyMemoryView_FromObject(obj); + if (view == NULL) { + return NULL; + } + + Py_buffer *buffer = PyMemoryView_GET_BUFFER(view); + if (buffer->itemsize != 1 || !PyBuffer_IsContiguous(buffer, 'C')) { + Py_DECREF(view); + PyErr_SetString(PyExc_TypeError, + "subject buffer must be a contiguous sequence of single-byte items"); + return NULL; + } + + *data_out = (const char *)buffer->buf; + *length_out = buffer->len; + return view; +} + PyObject * bytes_from_text(PyObject *obj) { diff --git a/tests/test_mmap.py b/tests/test_mmap.py new file mode 100644 index 0000000..15e3fee --- /dev/null +++ b/tests/test_mmap.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: 2025 ModelCloud.ai +# SPDX-FileCopyrightText: 2025 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +import gc +import mmap +import os +import tempfile +import unittest + +import pcre + + +class TestMmapSubject(unittest.TestCase): + def setUp(self): + fd, self.path = tempfile.mkstemp() + os.close(fd) + self.data = b"prefix\x00foo\xffbar123baz needle-42 tail\n" * 50 + with open(self.path, "wb") as f: + f.write(self.data) + self._fp = open(self.path, "rb") + self.mm = mmap.mmap(self._fp.fileno(), 0, access=mmap.ACCESS_READ) + + def tearDown(self): + gc.collect() + self.mm.close() + self._fp.close() + os.unlink(self.path) + + def test_search_returns_bytes_groups(self): + pattern = pcre.compile(rb"needle-(\d+)") + match = pattern.search(self.mm) + self.assertIsNotNone(match) + self.assertEqual(match.group(0), b"needle-42") + self.assertEqual(match.group(1), b"42") + self.assertIsInstance(match.group(0), bytes) + + def test_match_at_start(self): + pattern = pcre.compile(rb"prefix") + match = pattern.match(self.mm) + self.assertIsNotNone(match) + self.assertEqual(match.span(), (0, 6)) + + def test_fullmatch_on_exact_span(self): + pattern = pcre.compile(rb"prefix") + small = mmap.mmap(-1, len(b"prefix")) + small.write(b"prefix") + try: + match = pattern.fullmatch(small) + self.assertIsNotNone(match) + self.assertEqual(match.span(), (0, 6)) + del match + finally: + small.close() + + def test_finditer_matches_same_as_bytes(self): + pattern = pcre.compile(rb"needle-(\d+)") + from_mmap = [m.span() for m in pattern.finditer(self.mm)] + from_bytes = [m.span() for m in pattern.finditer(self.data)] + self.assertEqual(from_mmap, from_bytes) + self.assertEqual(len(from_mmap), 50) + + def test_findall(self): + pattern = pcre.compile(rb"needle-(\d+)") + self.assertEqual(pattern.findall(self.mm), [b"42"] * 50) + + def test_module_level_search(self): + match = pcre.search(rb"foo\xffbar", self.mm) + self.assertIsNotNone(match) + self.assertEqual(match.group(0), b"foo\xffbar") + + def test_pos_and_endpos_are_byte_offsets(self): + pattern = pcre.compile(rb"prefix") + first_end = pattern.match(self.mm).end() + self.assertIsNone(pattern.search(self.mm, pos=1, endpos=first_end - 1)) + self.assertIsNotNone(pattern.search(self.mm, pos=0, endpos=first_end)) + + def test_split_on_mmap(self): + pattern = pcre.compile(rb"\s+") + parts = pattern.split(self.mm[:20]) + self.assertTrue(all(isinstance(p, bytes) for p in parts)) + + def test_sub_on_mmap(self): + pattern = pcre.compile(rb"needle-\d+") + result = pattern.sub(b"X", self.mm) + self.assertNotIn(b"needle-42", result) + self.assertEqual(result.count(b"X"), 50) + + def test_noncontiguous_buffer_rejected(self): + # pcre.Pattern materializes plain `memoryview` subjects via + # .tobytes() before reaching the C extension (a pre-existing, + # unrelated compatibility shim), so exercise the low-level binding + # directly to confirm it refuses a non-contiguous buffer instead of + # silently reading the wrong bytes. + import array + + import pcre_ext_c as raw + + strided = memoryview(array.array("b", range(20)))[::2] + self.assertFalse(strided.contiguous) + pattern = raw.compile(rb"x") + with self.assertRaises((TypeError, BufferError)): + pattern.search(strided) + + def test_non_buffer_subject_still_rejected(self): + pattern = pcre.compile(rb"x") + with self.assertRaises(TypeError): + pattern.search(12345) + + def test_mmap_cannot_close_while_match_alive(self): + fd, path = tempfile.mkstemp() + os.close(fd) + try: + with open(path, "wb") as f: + f.write(b"hello world") + fp = open(path, "rb") + mm = mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ) + pattern = pcre.compile(rb"hello") + match = pattern.search(mm) + self.assertIsNotNone(match) + with self.assertRaises(BufferError): + mm.close() + del match + gc.collect() + mm.close() + fp.close() + finally: + os.unlink(path) + + def test_bytearray_subject_zero_copy_path(self): + buf = bytearray(b"hello bytearray world") + pattern = pcre.compile(rb"bytearray") + match = pattern.search(buf) + self.assertIsNotNone(match) + self.assertEqual(match.group(0), b"bytearray") + + +if __name__ == "__main__": + unittest.main() From 8078c27b646540bc3a71dae498c4f996754235f5 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 27 Jul 2026 04:58:35 +0000 Subject: [PATCH 2/3] chore(release): bump version to 0.5.0 --- README.md | 1 + pcre_ext/pcre2.c | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 033e42a..7eb1a2d 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 +* 07/27/2026 [0.5.0](https://github.com/ModelCloud/PyPcre/releases/tag/v0.5.0): Zero-copy buffer-protocol subject support (`mmap.mmap`, `bytearray`, `array.array`) with UTF-8 validation and GIL=0-safe memory pinning. 🗂️⚡ * 07/24/2026 [0.4.0](https://github.com/ModelCloud/PyPcre/releases/tag/v0.4.0): C extension hardening (memory/pointer safety, bounds checks, atomic allocator init), GIL=0 safety verified, vectorized UTF-8 index/offset conversion, GIL-release threshold for small calls, C `findall` implementation, and README competitor benchmarks. 🛡️⚡ * 04/13/2026 [0.3.0](https://github.com/ModelCloud/PyPcre/releases/tag/v0.3.0): Lower-overhead public `Match` objects, faster hot-path `match()` / `search()` / `fullmatch()` / `findall()`, and tighter free-threaded execution. ⚡ * 03/22/2026 [0.2.15](https://github.com/ModelCloud/PyPcre/releases/tag/v0.2.15): Python 3.15 `re` compatibility (`prefixmatch`, `NOFLAG`) ✅ diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index dbc1d9a..c890130 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -2739,7 +2739,7 @@ PyInit_pcre_ext_c(void) goto error_cache; } - if (PyModule_AddStringConstant(module, "__version__", "0.4.0") < 0) { + if (PyModule_AddStringConstant(module, "__version__", "0.5.0") < 0) { goto error_cache; } diff --git a/pyproject.toml b/pyproject.toml index 0df83f2..01795d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "PyPcre" -version = "0.4.0" +version = "0.5.0" description = "Modern, GIL-friendly, Fast Python bindings for PCRE2 with auto caching and JIT of compiled patterns." readme = "README.md" requires-python = ">=3.9" From 971044996d8ab0e92a0b3a796c1917aad635b407 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 27 Jul 2026 05:01:18 +0000 Subject: [PATCH 3/3] test(mmap): split directly on mmap instead of bytes slice --- tests/test_mmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_mmap.py b/tests/test_mmap.py index 15e3fee..339da18 100644 --- a/tests/test_mmap.py +++ b/tests/test_mmap.py @@ -78,7 +78,7 @@ def test_pos_and_endpos_are_byte_offsets(self): def test_split_on_mmap(self): pattern = pcre.compile(rb"\s+") - parts = pattern.split(self.mm[:20]) + parts = pattern.split(self.mm) self.assertTrue(all(isinstance(p, bytes) for p in parts)) def test_sub_on_mmap(self):