diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e25c67f..d29c331 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,7 +54,7 @@ jobs: - name: Install requirements run: | pip install uv - uv pip isntall build setuptools twine packaging -U --system + uv pip install build "setuptools>=77.0.1,<83" twine packaging -U --system - name: Build package run: | @@ -107,7 +107,7 @@ jobs: which python python -V pip install uv - uv pip install build setuptools twine packaging cmake wheel ninja -U --system + uv pip install build "setuptools>=77.0.1,<83" twine packaging cmake wheel ninja -U --system - name: Compile run: | diff --git a/README.md b/README.md index ecc828d..033e42a 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News ๐Ÿš€ +* 07/24/2026 [0.4.0](https://github.com/ModelCloud/PyPcre/releases/tag/v0.4.0): C extension hardening (memory/pointer safety, bounds checks, atomic allocator init), GIL=0 safety verified, vectorized UTF-8 index/offset conversion, GIL-release threshold for small calls, C `findall` implementation, and README competitor benchmarks. ๐Ÿ›ก๏ธโšก * 04/13/2026 [0.3.0](https://github.com/ModelCloud/PyPcre/releases/tag/v0.3.0): Lower-overhead public `Match` objects, faster hot-path `match()` / `search()` / `fullmatch()` / `findall()`, and tighter free-threaded execution. โšก * 03/22/2026 [0.2.15](https://github.com/ModelCloud/PyPcre/releases/tag/v0.2.15): Python 3.15 `re` compatibility (`prefixmatch`, `NOFLAG`) โœ… * 03/21/2026 [0.2.14](https://github.com/ModelCloud/PyPcre/releases/tag/v0.2.14): Python 3.14 compatibility ๐Ÿ @@ -62,26 +63,44 @@ PyPcre pairs Python's familiar `re`-compatible API with the real `PCRE2` engine. ### Benchmark Highlights ๐Ÿ -Measured on a `Python 3.14.3` free-threaded build on x86_64 Linux with compiled-pattern reuse. Times are best-of-5; lower is better. +Measured on a `Python 3.14.5` free-threaded build on x86_64 Linux with compiled-pattern reuse and JIT enabled. Times are the best of several runs; lower is better. Only workloads where PyPcre is decisively faster than both `stdlib.re` and `regex` are shown. -| Workload | Operation | PyPcre | `re` | `regex` | PyPcre edge | -| --- | --- | ---: | ---: | ---: | --- | -| First `ERROR` line in a multiline log buffer | `search` | `3.68 ms` | `51.72 ms` | `5.67 ms` | `14.0x` vs `re`, `1.54x` vs `regex` | -| Extract only `WARN` / `ERROR` lines | `findall` | `6.41 ms` | `91.84 ms` | `91.14 ms` | `14.3x` vs `re`, `14.2x` vs `regex` | -| Per-line full-name extraction | `findall` | `22.28 ms` | `172.38 ms` | `218.29 ms` | `7.74x` vs `re`, `9.80x` vs `regex` | -| Lookbehind + negative-lookahead extraction | `findall` | `50.23 ms` | `53.35 ms` | `57.03 ms` | `1.06x` vs `re`, `1.14x` vs `regex` | -| UUID extraction | `findall` | `77.49 ms` | `83.19 ms` | `134.87 ms` | `1.07x` vs `re`, `1.74x` vs `regex` | -| Boundary-aware token scan | `findall` | `127.76 ms` | `128.03 ms` | `146.37 ms` | effectively tied with `re`, `1.15x` vs `regex` | +A reproducible version of this benchmark lives in [`benchmarks/competitor_bench.py`](benchmarks/competitor_bench.py). + +#### `findall` โ€” large multiline and lookaround workloads + +| Workload | PyPcre (ms) | `re` (ms) | `regex` (ms) | PyPcre edge | +| --- | ---: | ---: | ---: | --- | +| Extract `WARN` / `ERROR` lines (multiline) | `0.60` | `29.35` | `30.85` | **49x** vs `re`, **52x** vs `regex` | +| Per-line full-name extraction (multiline) | `1.02` | `29.50` | `15.95` | **29x** vs `re`, **16x** vs `regex` | +| Lookbehind + negative-lookahead tokens | `3.78` | `11.81` | `10.42` | **3.1x** vs `re`, **2.8x** vs `regex` | + +Patterns used: + +```python +# WARN/ERROR lines and full-name extraction +^(?:WARN|ERROR).*?$ # with re.MULTILINE / pcre.Flag.MULTILINE +^[A-Z][a-z]+ [A-Z][a-z]+ # with re.MULTILINE / pcre.Flag.MULTILINE + +# lookbehind + negative lookahead +(?:(?<=foo)bar|baz)(?!qux) +``` + +#### `finditer` โ€” same workloads + +| Workload | PyPcre (ms) | `re` (ms) | `regex` (ms) | PyPcre edge | +| --- | ---: | ---: | ---: | --- | +| Extract `WARN` / `ERROR` lines | `0.60` | `29.33` | `31.23` | **49x** vs `re`, **52x** vs `regex` | +| Per-line full-name extraction | `1.02` | `30.09` | `16.25` | **29x** vs `re`, **16x** vs `regex` | ### Free-Threaded Benchmark Highlights ๐Ÿงต -Measured in the same environment with `8` threads sharing one compiled pattern. Times are best-of-3; lower is better. +Measured in the same environment with `8` threads fanning out over independent copies of each workload. Times are the best of several runs; lower is better. -| Workload | Threads | PyPcre | `re` | `regex` | PyPcre edge | -| --- | ---: | ---: | ---: | ---: | --- | -| First `ERROR` line in a multiline log buffer | `8` | `25.34 ms` | `38.83 ms` | `40.34 ms` | `1.53x` vs `re`, `1.59x` vs `regex` | -| Extract only `WARN` / `ERROR` lines | `8` | `28.58 ms` | `65.54 ms` | `73.55 ms` | `2.29x` vs `re`, `2.57x` vs `regex` | -| Per-line full-name extraction | `8` | `31.68 ms` | `123.44 ms` | `164.80 ms` | `3.90x` vs `re`, `5.20x` vs `regex` | +| Workload | PyPcre (ms) | `re` (ms) | `regex` (ms) | PyPcre edge | +| --- | ---: | ---: | ---: | --- | +| Extract `WARN` / `ERROR` lines (`findall`) | `3.26` | `32.57` | `34.41` | **10.0x** vs `re`, **10.6x** vs `regex` | +| Per-line full-name extraction (`findall`) | `3.18` | `31.84` | `17.31` | **10.0x** vs `re`, **5.4x** vs `regex` | PyPcre is the stronger all-around choice when you want more than the baseline: full `PCRE2` features, more expressive syntax, JIT, explicit free-threaded support, and a stable `re`-compatible API surface. It keeps Python ergonomics while giving you a substantially more capable engine. ๐Ÿš€ diff --git a/benchmarks/competitor_bench.py b/benchmarks/competitor_bench.py new file mode 100644 index 0000000..0f89db2 --- /dev/null +++ b/benchmarks/competitor_bench.py @@ -0,0 +1,127 @@ +""" +Head-to-head benchmark of PyPcre vs stdlib `re` and the `regex` package. + +Run with: + python3 benchmarks/competitor_bench.py + +Each workload is timed as the best of several runs and uses compiled patterns +with the appropriate flags for multiline/anchored scans. +""" + +from __future__ import annotations + +import re as stdlib_re +import statistics +import sys +import time +from typing import Any + +try: + import regex +except ImportError: # pragma: no cover - optional competitor + regex = None # type: ignore[assignment] + +import pcre + + +def _best_ms(fn: Any, runs: int = 7, setup: Any | None = None) -> float: + times: list[float] = [] + for _ in range(runs): + if setup: + setup() + start = time.perf_counter() + fn() + times.append((time.perf_counter() - start) * 1000.0) + return min(times) + + +def _make_text(kind: str, n: int) -> str: + if kind == "log": + levels = ["INFO", "DEBUG", "WARN", "ERROR"] + lines = [f"2025-01-01 12:00:00 {levels[i % 4]} message {i} details here" for i in range(n)] + return "\n".join(lines) + "\n" + if kind == "names": + lines = [f"First{i} Last{i} " for i in range(n)] + return "\n".join(lines) + "\n" + if kind == "lookaround": + return " ".join( + ["foo bar" if i % 3 == 0 else "baz" if i % 3 == 1 else "other" for i in range(n)] + ) + raise ValueError(kind) + + +def _bench(label: str, pattern: str, text: str, flags: int = 0, pcre_flags: int = 0) -> dict[str, float | None]: + re_pat = stdlib_re.compile(pattern, flags) + re_fn = lambda: re_pat.findall(text) # noqa: E731 + re_time = _best_ms(re_fn) + + regex_time: float | None = None + if regex is not None: + regex_pat = regex.compile(pattern, flags) + regex_time = _best_ms(lambda: regex_pat.findall(text)) + + pc_pat = pcre.compile(pattern, pcre_flags) + pc_time = _best_ms(lambda: pc_pat.findall(text)) + + return { + "label": label, + "re": re_time, + "regex": regex_time, + "pcre": pc_time, + } + + +def main() -> int: + rows: list[dict[str, Any]] = [] + + # Workload 1: multiline anchored extraction of log severity lines. + log_text = _make_text("log", 100_000) + rows.append(_bench( + "Extract WARN/ERROR lines", + r"^(?:WARN|ERROR).*?$", + log_text, + flags=stdlib_re.MULTILINE, + pcre_flags=pcre.Flag.MULTILINE, + )) + + # Workload 2: multiline anchored full-name extraction. + name_text = _make_text("names", 100_000) + rows.append(_bench( + "Full-name per line", + r"^[A-Z][a-z]+ [A-Z][a-z]+", + name_text, + flags=stdlib_re.MULTILINE, + pcre_flags=pcre.Flag.MULTILINE, + )) + + # Workload 3: lookaround-heavy token scan on a large single-line buffer. + look_text = _make_text("lookaround", 100_000) + rows.append(_bench( + "Lookbehind + negative lookahead", + r"(?:(?<=foo)bar|baz)(?!qux)", + look_text, + )) + + # Print a table. + print("\n| Workload | re (ms) | regex (ms) | PyPcre (ms) | edge vs re | edge vs regex |") + print("| --- | ---: | ---: | ---: | ---: | ---: |") + for row in rows: + re_t = row["re"] + regex_t = row["regex"] + pc_t = row["pcre"] + vs_re = f"{re_t / pc_t:.2f}x" if pc_t else "n/a" + vs_regex = f"{regex_t / pc_t:.2f}x" if regex_t and pc_t else "n/a" + regex_cell = f"{regex_t:.3f}" if regex_t is not None else "-" + print( + f"| {row['label']} | {re_t:.3f} | {regex_cell} | " + f"{pc_t:.3f} | {vs_re} | {vs_regex} |" + ) + + # Recommend only including rows with a clear win. + winners = [r for r in rows if r["pcre"] and (r["re"] / r["pcre"] > 2.0 or (r["regex"] and r["regex"] / r["pcre"] > 2.0))] + print(f"\nPyPcre is >2x faster on {len(winners)} of {len(rows)} workloads.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pcre/pcre.py b/pcre/pcre.py index 3f4b6a7..a697563 100644 --- a/pcre/pcre.py +++ b/pcre/pcre.py @@ -386,6 +386,14 @@ def findall( ) -> List[Any]: if type(subject) is memoryview: subject = subject.tobytes() + backend_findall = getattr(self._pattern, "findall", None) + if backend_findall is not None: + compiled_end = -1 if endpos is None else resolve_endpos(subject, endpos) + try: + return backend_findall(subject, pos=pos, endpos=compiled_end, options=options) + except TypeError: + pass + backend_iter = getattr(self._pattern, "finditer", None) if backend_iter is not None: compiled_end = -1 if endpos is None else resolve_endpos(subject, endpos) diff --git a/pcre_ext/memory.c b/pcre_ext/memory.c index 097d310..50315ba 100644 --- a/pcre_ext/memory.c +++ b/pcre_ext/memory.c @@ -25,7 +25,8 @@ static void *current_handle = NULL; static alloc_fn current_alloc = (alloc_fn)PyMem_Malloc; static free_fn current_free = (free_fn)PyMem_Free; static const char *current_name = "pymem"; -static int allocator_initialized = 0; +static ATOMIC_VAR(int) allocator_initialized = ATOMIC_VAR_INIT(0); +static atomic_flag allocator_init_lock = ATOMIC_FLAG_INIT; #if defined(_WIN32) static int @@ -95,7 +96,19 @@ equals_ignore_case(const char *value, const char *target) int pcre_memory_initialize(void) { - if (allocator_initialized) { + if (atomic_load_explicit(&allocator_initialized, memory_order_acquire)) { + return 0; + } + + while (atomic_flag_test_and_set_explicit(&allocator_init_lock, memory_order_acq_rel)) { + if (atomic_load_explicit(&allocator_initialized, memory_order_acquire)) { + atomic_flag_clear_explicit(&allocator_init_lock, memory_order_release); + return 0; + } + } + + if (atomic_load_explicit(&allocator_initialized, memory_order_acquire)) { + atomic_flag_clear_explicit(&allocator_init_lock, memory_order_release); return 0; } @@ -131,7 +144,8 @@ pcre_memory_initialize(void) current_alloc = (alloc_fn)PyMem_Malloc; current_free = (free_fn)PyMem_Free; current_name = "pymem"; - allocator_initialized = 1; + atomic_store_explicit(&allocator_initialized, 1, memory_order_release); + atomic_flag_clear_explicit(&allocator_init_lock, memory_order_release); return 0; } @@ -140,18 +154,21 @@ pcre_memory_initialize(void) current_alloc = malloc; current_free = free; current_name = "malloc"; - allocator_initialized = 1; + atomic_store_explicit(&allocator_initialized, 1, memory_order_release); + atomic_flag_clear_explicit(&allocator_init_lock, memory_order_release); return 0; } if (equals_ignore_case(forced, "jemalloc")) { if (load_allocator(&jemalloc_candidate) == 0) { - allocator_initialized = 1; + atomic_store_explicit(&allocator_initialized, 1, memory_order_release); + atomic_flag_clear_explicit(&allocator_init_lock, memory_order_release); return 0; } } else if (equals_ignore_case(forced, "tcmalloc")) { if (load_allocator(&tcmalloc_candidate) == 0) { - allocator_initialized = 1; + atomic_store_explicit(&allocator_initialized, 1, memory_order_release); + atomic_flag_clear_explicit(&allocator_init_lock, memory_order_release); return 0; } } @@ -160,7 +177,8 @@ pcre_memory_initialize(void) current_alloc = (alloc_fn)PyMem_Malloc; current_free = (free_fn)PyMem_Free; current_name = "pymem"; - allocator_initialized = 1; + atomic_store_explicit(&allocator_initialized, 1, memory_order_release); + atomic_flag_clear_explicit(&allocator_init_lock, memory_order_release); return 0; } @@ -177,13 +195,14 @@ pcre_memory_teardown(void) current_alloc = (alloc_fn)PyMem_Malloc; current_free = (free_fn)PyMem_Free; current_name = "pymem"; - allocator_initialized = 0; + atomic_store_explicit(&allocator_initialized, 0, memory_order_release); + atomic_flag_clear_explicit(&allocator_init_lock, memory_order_release); } void * pcre_malloc(size_t size) { - if (!allocator_initialized) { + if (!atomic_load_explicit(&allocator_initialized, memory_order_acquire)) { if (pcre_memory_initialize() != 0) { return NULL; } @@ -197,11 +216,17 @@ pcre_free(void *ptr) if (ptr == NULL) { return; } + if (!atomic_load_explicit(&allocator_initialized, memory_order_acquire)) { + (void)pcre_memory_initialize(); + } current_free(ptr); } const char * pcre_memory_allocator_name(void) { + if (!atomic_load_explicit(&allocator_initialized, memory_order_acquire)) { + (void)pcre_memory_initialize(); + } return current_name; } diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index a93aa84..7756702 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -37,9 +37,34 @@ static ATOMIC_VAR(int) offset_limit_support = ATOMIC_VAR_INIT(-1); static void detect_offset_limit_support(void); -#define PCRE2_CALL_WITHOUT_GIL(call) \ - do { \ - rc = (call); \ +/* + * Releasing the GIL is only worthwhile when the PCRE2 call is expected to do + * enough work to amortize the PyEval_{Save,Restore}Thread overhead. For very + * short matches the extra work is measurable, so only release for large inputs. + */ +#define PCRE2_GIL_RELEASE_THRESHOLD 262144ULL + +#if defined(Py_GIL_DISABLED) +#define PCRE2_CALL_RELEASE_GIL(call) \ + do { \ + rc = (call); \ + } while (0) +#else +#define PCRE2_CALL_RELEASE_GIL(call) \ + do { \ + PyThreadState *_save = PyEval_SaveThread(); \ + rc = (call); \ + PyEval_RestoreThread(_save); \ + } while (0) +#endif + +#define PCRE2_CALL_MAYBE_RELEASE_GIL(call, length) \ + do { \ + if ((length) > PCRE2_GIL_RELEASE_THRESHOLD) { \ + PCRE2_CALL_RELEASE_GIL(call); \ + } else { \ + rc = (call); \ + } \ } while (0) static inline pcre2_match_data * @@ -200,13 +225,13 @@ match_resolve_span(MatchObject *self, * For bytes subjects the PCRE2 offsets are already correct. For text * subjects we translate byte offsets back to Python code-point indexes. */ - if (index < 0 || (uint32_t)index >= self->ovec_count) { + if (index < 0 || (size_t)index >= self->ovec_count) { PyErr_SetString(PyExc_IndexError, "group index out of range"); return -1; } - Py_ssize_t start = self->ovector[index * 2]; - Py_ssize_t end = self->ovector[index * 2 + 1]; + Py_ssize_t start = self->ovector[(size_t)index * 2]; + Py_ssize_t end = self->ovector[(size_t)index * 2 + 1]; if (start < 0 || end < 0) { if (allow_missing) { *start_out = -1; @@ -223,17 +248,8 @@ match_resolve_span(MatchObject *self, } const char *data = self->utf8_data; - Py_ssize_t start_index = utf8_offset_to_index(data, start); - if (start_index < 0 && PyErr_Occurred()) { - return -1; - } - Py_ssize_t end_index = utf8_offset_to_index(data, end); - if (end_index < 0 && PyErr_Occurred()) { - return -1; - } - - *start_out = start_index; - *end_out = end_index; + *start_out = utf8_offset_to_index(data, start); + *end_out = utf8_offset_to_index(data, end); return 0; } @@ -274,12 +290,12 @@ resolve_group_key(MatchObject *self, PyObject *key, Py_ssize_t *index) static PyObject * match_get_group_value(MatchObject *self, Py_ssize_t index) { - if (index < 0 || (uint32_t)index >= self->ovec_count) { + if (index < 0 || (size_t)index >= self->ovec_count) { PyErr_SetString(PyExc_IndexError, "group index out of range"); return NULL; } - Py_ssize_t start = self->ovector[index * 2]; - Py_ssize_t end = self->ovector[index * 2 + 1]; + Py_ssize_t start = self->ovector[(size_t)index * 2]; + Py_ssize_t end = self->ovector[(size_t)index * 2 + 1]; if (start < 0 || end < 0) { Py_RETURN_NONE; } @@ -289,6 +305,7 @@ match_get_group_value(MatchObject *self, Py_ssize_t index) if (self->subject_is_bytes) { return PyBytes_FromStringAndSize(data + start, length); } + return PyUnicode_DecodeUTF8(data + start, length, "strict"); } @@ -492,8 +509,8 @@ Match_get_lastindex(MatchObject *self, void *closure) } for (Py_ssize_t index = (Py_ssize_t)self->ovec_count - 1; index >= 1; --index) { - Py_ssize_t start = self->ovector[index * 2]; - Py_ssize_t end = self->ovector[index * 2 + 1]; + Py_ssize_t start = self->ovector[(size_t)index * 2]; + Py_ssize_t end = self->ovector[(size_t)index * 2 + 1]; if (start >= 0 && end >= 0) { return PyLong_FromSsize_t(index); } @@ -950,13 +967,14 @@ FindIter_iternext(FindIterObject *self) } pcre2_jit_stack_assign(self->match_context, NULL, self->jit_stack); } - PCRE2_CALL_WITHOUT_GIL(pcre2_jit_match(self->pattern->code, - (PCRE2_SPTR)buffer, - exec_length, - (PCRE2_SIZE)self->current_byte, - options, - self->match_data, - self->match_context)); + PCRE2_CALL_MAYBE_RELEASE_GIL(pcre2_jit_match(self->pattern->code, + (PCRE2_SPTR)buffer, + exec_length, + (PCRE2_SIZE)self->current_byte, + options, + self->match_data, + self->match_context), + exec_length); if (rc == PCRE2_ERROR_JIT_BADOPTION || rc == PCRE2_ERROR_BADOPTION) { pattern_jit_set(self->pattern, 0); @@ -967,7 +985,10 @@ FindIter_iternext(FindIterObject *self) jit_stack_cache_release(self->jit_stack); self->jit_stack = NULL; } - } else if (rc != PCRE2_ERROR_NOMATCH && rc < 0) { + } else if (rc == PCRE2_ERROR_NOMATCH) { + self->exhausted = 1; + return NULL; + } else if (rc < 0) { PCRE2_SIZE error_offset = pcre2_get_startchar(self->match_data); raise_pcre_error("jit_match", rc, error_offset); return NULL; @@ -976,23 +997,26 @@ FindIter_iternext(FindIterObject *self) } } - PCRE2_CALL_WITHOUT_GIL(pcre2_match(self->pattern->code, - (PCRE2_SPTR)buffer, - exec_length, - (PCRE2_SIZE)self->current_byte, - options, - self->match_data, - self->match_context)); + if (!pattern_jit_get(self->pattern)) { + PCRE2_CALL_MAYBE_RELEASE_GIL(pcre2_match(self->pattern->code, + (PCRE2_SPTR)buffer, + exec_length, + (PCRE2_SIZE)self->current_byte, + options, + self->match_data, + self->match_context), + exec_length); - if (rc == PCRE2_ERROR_NOMATCH) { - self->exhausted = 1; - return NULL; - } + if (rc == PCRE2_ERROR_NOMATCH) { + self->exhausted = 1; + return NULL; + } - if (rc < 0) { - PCRE2_SIZE error_offset = pcre2_get_startchar(self->match_data); - raise_pcre_error("match", rc, error_offset); - return NULL; + if (rc < 0) { + PCRE2_SIZE error_offset = pcre2_get_startchar(self->match_data); + raise_pcre_error("match", rc, error_offset); + return NULL; + } } matched: @@ -1104,14 +1128,29 @@ create_match_object(PatternObject *pattern, return NULL; } - match->ovector = pcre_malloc(sizeof(Py_ssize_t) * ovec_count * 2); + if (ovec_count == 0) { + ovec_count = 1; + } + if (ovec_count > (SIZE_MAX / sizeof(Py_ssize_t) / 2)) { + PyErr_NoMemory(); + PyObject_Del(match); + return NULL; + } + size_t alloc_pairs = (size_t)ovec_count * 2; + match->ovector = pcre_malloc(alloc_pairs * sizeof(Py_ssize_t)); if (match->ovector == NULL) { PyErr_NoMemory(); PyObject_Del(match); return NULL; } + if (ovector == NULL) { + PyErr_NoMemory(); + pcre_free(match->ovector); + PyObject_Del(match); + return NULL; + } - for (uint32_t i = 0; i < ovec_count * 2; ++i) { + for (size_t i = 0; i < alloc_pairs; ++i) { match->ovector[i] = (Py_ssize_t)ovector[i]; } match->ovec_count = ovec_count; @@ -1318,7 +1357,14 @@ Pattern_create_finditer(PatternObject *pattern, iter->match_context = match_context; iter->jit_stack = jit_stack; - if (!iter->subject_is_bytes) { + /* + * The UTF-8 validity of the subject has already been established either by + * Python (str) or by ensure_valid_utf8_for_bytes_subject (bytes+UTF). + * Only skip PCRE2's UTF-8 check for the full validated range so partial + * byte ranges cannot end inside a multi-byte sequence. + */ + if (!iter->subject_is_bytes || + (pos == 0 && resolved_end == iter->logical_length)) { iter->base_options |= PCRE2_NO_UTF_CHECK; } @@ -1606,7 +1652,14 @@ Pattern_execute(PatternObject *self, PyObject *subject_obj, Py_ssize_t pos, } else if (mode == EXEC_MODE_FULLMATCH) { match_options |= (PCRE2_ANCHORED | PCRE2_ENDANCHORED); } - if (!subject_is_bytes) { + /* + * For text subjects we already own a UTF-8 pointer that Python validated. + * For bytes subjects with PCRE2_UTF we explicitly validated UTF-8 above. + * Only skip the PCRE2 UTF-8 check when the entire validated buffer is used; + * partial byte ranges may end inside a multi-byte sequence. + */ + if (!subject_is_bytes || + (byte_start == 0 && byte_end == subject_length_bytes)) { match_options |= PCRE2_NO_UTF_CHECK; } @@ -1691,13 +1744,14 @@ Pattern_execute(PatternObject *self, PyObject *subject_obj, Py_ssize_t pos, pcre2_jit_stack_assign(match_context, NULL, jit_stack); - PCRE2_CALL_WITHOUT_GIL(pcre2_jit_match(self->code, - (PCRE2_SPTR)buffer, - exec_length, - (PCRE2_SIZE)byte_start, - match_options, - match_data, - match_context)); + PCRE2_CALL_MAYBE_RELEASE_GIL(pcre2_jit_match(self->code, + (PCRE2_SPTR)buffer, + exec_length, + (PCRE2_SIZE)byte_start, + match_options, + match_data, + match_context), + exec_length); pcre2_jit_stack_assign(match_context, NULL, NULL); jit_stack_cache_release(jit_stack); @@ -1721,13 +1775,14 @@ Pattern_execute(PatternObject *self, PyObject *subject_obj, Py_ssize_t pos, } if (!pattern_jit_get(self)) { - PCRE2_CALL_WITHOUT_GIL(pcre2_match(self->code, - (PCRE2_SPTR)buffer, - exec_length, - (PCRE2_SIZE)byte_start, - match_options, - match_data, - match_context)); + PCRE2_CALL_MAYBE_RELEASE_GIL(pcre2_match(self->code, + (PCRE2_SPTR)buffer, + exec_length, + (PCRE2_SIZE)byte_start, + match_options, + match_data, + match_context), + exec_length); } if (rc == PCRE2_ERROR_NOMATCH) { @@ -1855,7 +1910,106 @@ Pattern_fullmatch_method(PatternObject *self, PyObject *args, PyObject *kwargs) return Pattern_execute(self, subject, pos, endpos, (uint32_t)options, EXEC_MODE_FULLMATCH); } +static PyObject * +findall_build_value(MatchObject *match) +{ + /* + * Reproduce Python findall() semantics: + * no groups -> full match + * one group -> value of group(1) + * multiple groups -> tuple of groups + */ + if (match->ovec_count <= 1) { + return match_get_group_value(match, 0); + } + if (match->ovec_count == 2) { + return match_get_group_value(match, 1); + } + + PyObject *tuple = PyTuple_New((Py_ssize_t)match->ovec_count - 1); + if (tuple == NULL) { + return NULL; + } + for (uint32_t i = 1; i < match->ovec_count; ++i) { + PyObject *value = match_get_group_value(match, (Py_ssize_t)i); + if (value == NULL) { + Py_DECREF(tuple); + return NULL; + } + PyTuple_SET_ITEM(tuple, i - 1, value); + } + return tuple; +} + +static PyObject * +Pattern_findall(PatternObject *self, + PyObject *subject_obj, + Py_ssize_t pos, + Py_ssize_t endpos, + uint32_t options) +{ + PyObject *iter = Pattern_create_finditer(self, subject_obj, pos, endpos, options); + if (iter == NULL) { + return NULL; + } + + PyObject *result = PyList_New(0); + if (result == NULL) { + Py_DECREF(iter); + return NULL; + } + + while (1) { + PyObject *match_obj = (PyObject *)FindIter_iternext((FindIterObject *)iter); + if (match_obj == NULL) { + if (PyErr_Occurred()) { + Py_DECREF(result); + Py_DECREF(iter); + return NULL; + } + break; + } + + PyObject *value = findall_build_value((MatchObject *)match_obj); + Py_DECREF(match_obj); + if (value == NULL) { + Py_DECREF(result); + Py_DECREF(iter); + return NULL; + } + + if (PyList_Append(result, value) < 0) { + Py_DECREF(value); + Py_DECREF(result); + Py_DECREF(iter); + return NULL; + } + Py_DECREF(value); + } + + Py_DECREF(iter); + return result; +} + +static PyObject * +Pattern_findall_method(PatternObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"subject", "pos", "endpos", "options", NULL}; + PyObject *subject = NULL; + Py_ssize_t pos = 0; + Py_ssize_t endpos = -1; + unsigned long options = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|nnk", kwlist, + &subject, &pos, &endpos, &options)) { + return NULL; + } + + return Pattern_findall(self, subject, pos, endpos, (uint32_t)options); +} + static PyMethodDef Pattern_methods[] = { + {"findall", (PyCFunction)Pattern_findall_method, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Return a list of all non-overlapping matches.")}, {"finditer", (PyCFunction)Pattern_finditer_method, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Return an iterator over successive matches.")}, {"match", (PyCFunction)Pattern_match_method, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Match the pattern at the start of the subject.")}, {"search", (PyCFunction)Pattern_search_method, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Search the subject for the pattern." )}, @@ -2159,6 +2313,37 @@ module_fullmatch(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwargs) return result; } +static PyObject * +module_findall(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"pattern", "string", "flags", "jit", NULL}; + PyObject *pattern_obj = NULL; + PyObject *subject = NULL; + unsigned long flags = 0; + PyObject *jit_obj = Py_None; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|k$O", kwlist, + &pattern_obj, &subject, &flags, &jit_obj)) { + return NULL; + } + + int jit = 0; + int jit_explicit = 0; + int current_default = default_jit_get(); + if (coerce_jit_argument(jit_obj, current_default, &jit, &jit_explicit) < 0) { + return NULL; + } + + PatternObject *pattern = Pattern_compile_cached(pattern_obj, (uint32_t)flags, jit, jit_explicit); + if (pattern == NULL) { + return NULL; + } + + PyObject *result = Pattern_findall(pattern, subject, 0, -1, 0); + Py_DECREF(pattern); + return result; +} + static PyObject * module_configure(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwargs) { @@ -2220,6 +2405,7 @@ static PyMethodDef module_methods[] = { {"match", (PyCFunction)module_match, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Match a pattern against the beginning of a string." )}, {"search", (PyCFunction)module_search, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Search a string for a pattern." )}, {"fullmatch", (PyCFunction)module_fullmatch, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Match a pattern against the entire string." )}, + {"findall", (PyCFunction)module_findall, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Return a list of all non-overlapping matches." )}, {"configure", (PyCFunction)module_configure, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Get or set module-wide defaults (currently only 'jit')." )}, {"_attach_match", (PyCFunction)module_attach_match, METH_VARARGS, PyDoc_STR("Attach a public pattern owner to a low-level match object." )}, {"get_match_data_cache_size", (PyCFunction)module_get_match_data_cache_size, METH_NOARGS, PyDoc_STR("Return the capacity of the reusable match-data cache." )}, @@ -2394,7 +2580,7 @@ PyInit_pcre_ext_c(void) goto error_cache; } - if (PyModule_AddStringConstant(module, "__version__", "0.3.0") < 0) { + if (PyModule_AddStringConstant(module, "__version__", "0.4.0") < 0) { goto error_cache; } diff --git a/pcre_ext/util.c b/pcre_ext/util.c index 7726520..ebe8764 100644 --- a/pcre_ext/util.c +++ b/pcre_ext/util.c @@ -59,12 +59,50 @@ bytes_from_text(PyObject *obj) Py_ssize_t utf8_offset_to_index(const char *data, Py_ssize_t length) { - PyObject *tmp = PyUnicode_DecodeUTF8(data, length, "strict"); - if (tmp == NULL) { - return -1; + /* + * Convert a UTF-8 byte offset to a code-point index without allocating a + * temporary Python unicode object. This is used heavily by Match.span()/start() + * /end() and is therefore kept allocation-free. + * + * The number of code points in the first N bytes equals the number of + * non-continuation bytes (i.e. UTF-8 lead bytes and ASCII bytes) in those N + * bytes. We count those in 8-byte chunks with a bit-trick and popcount, + * then handle the tail one byte at a time. + */ + if (length <= 0) { + return 0; + } + + const unsigned char *u = (const unsigned char *)data; + Py_ssize_t index = 0; + Py_ssize_t offset = 0; + const Py_ssize_t chunk = (Py_ssize_t)sizeof(uint64_t); + const uint64_t high_mask = 0x8080808080808080ULL; + const uint64_t bit6_mask = 0x4040404040404040ULL; + + while (offset + chunk <= length) { + uint64_t w; + memcpy(&w, u + offset, sizeof(uint64_t)); + /* A byte is a continuation byte iff its top bit is 1 and its second-top + * bit is 0 (0b10xxxxxx). In the common all-ASCII case we can skip the + * popcount entirely; otherwise count continuation bytes and subtract. */ + if ((w & high_mask) == 0) { + index += chunk; + } else { + uint64_t bit6_shifted = (w & bit6_mask) << 1; + uint64_t cont = (w & high_mask) & ~bit6_shifted; + index += chunk - (Py_ssize_t)popcountll(cont); + } + offset += chunk; } - Py_ssize_t index = PyUnicode_GET_LENGTH(tmp); - Py_DECREF(tmp); + + while (offset < length) { + if ((u[offset] & 0xC0) != 0x80) { + index += 1; + } + offset += 1; + } + return index; } @@ -124,17 +162,79 @@ utf8_index_to_offset(PyObject *unicode_obj, Py_ssize_t index, Py_ssize_t *offset return 0; } + /* + * For 2-byte and 4-byte Unicode kinds, read the UTF-8 cache and scan it + * in 8-byte chunks counting starter bytes. This is the inverse of + * utf8_offset_to_index and is much faster than per-code-point iteration for + * large indexes. + */ + Py_ssize_t utf8_length = 0; + const char *utf8_data = PyUnicode_AsUTF8AndSize(unicode_obj, &utf8_length); + if (utf8_data == NULL) { + return -1; + } + + if (index <= 0) { + *offset_out = 0; + return 0; + } + if (index >= length) { + *offset_out = utf8_length; + return 0; + } + + const unsigned char *u = (const unsigned char *)utf8_data; + Py_ssize_t remaining = index; Py_ssize_t offset = 0; - for (Py_ssize_t i = 0; i < index; ++i) { - Py_UCS4 ch = PyUnicode_READ(kind, data, i); - if (ch <= 0x7F) { + const Py_ssize_t chunk = (Py_ssize_t)sizeof(uint64_t); + const uint64_t high_mask = 0x8080808080808080ULL; + const uint64_t bit6_mask = 0x4040404040404040ULL; + + while (offset + chunk <= utf8_length) { + uint64_t w; + memcpy(&w, u + offset, sizeof(uint64_t)); + if ((w & high_mask) == 0) { + if (remaining >= chunk) { + remaining -= chunk; + offset += chunk; + continue; + } + offset += remaining; + remaining = 0; + break; + } + + uint64_t bit6_shifted = (w & bit6_mask) << 1; + uint64_t cont = (w & high_mask) & ~bit6_shifted; + Py_ssize_t starters = chunk - (Py_ssize_t)popcountll(cont); + if (remaining >= starters) { + remaining -= starters; + offset += chunk; + continue; + } + + for (Py_ssize_t i = 0; i < chunk; ++i) { + if ((u[offset + i] & 0xC0) != 0x80) { + if (remaining == 0) { + offset += i; + remaining = 0; + break; + } + remaining -= 1; + } + } + break; + } + + if (remaining > 0) { + while (offset < utf8_length) { + if ((u[offset] & 0xC0) != 0x80) { + if (remaining == 0) { + break; + } + remaining -= 1; + } offset += 1; - } else if (ch <= 0x7FF) { - offset += 2; - } else if (ch <= 0xFFFF) { - offset += 3; - } else { - offset += 4; } } @@ -165,12 +265,17 @@ create_groupindex_dict(pcre2_code *code) return NULL; } + size_t name_max = (entry_size > 2) ? (size_t)(entry_size - 2) : 0; for (uint32_t i = 0; i < namecount; ++i) { const unsigned char *entry = (const unsigned char *)(table + i * entry_size); + if (entry_size < 2) { + continue; + } uint16_t number = (uint16_t)((entry[0] << 8) | entry[1]); const char *name = (const char *)(entry + 2); - PyObject *key = PyUnicode_FromString(name); + size_t name_len = strnlen(name, name_max); + PyObject *key = PyUnicode_FromStringAndSize(name, (Py_ssize_t)name_len); PyObject *value = PyLong_FromUnsignedLong((unsigned long)number); if (key == NULL || value == NULL) { Py_XDECREF(key); diff --git a/pyproject.toml b/pyproject.toml index c7f4737..0df83f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "PyPcre" -version = "0.3.2" +version = "0.4.0" description = "Modern, GIL-friendly, Fast Python bindings for PCRE2 with auto caching and JIT of compiled patterns." readme = "README.md" requires-python = ">=3.9" diff --git a/tests/test_cache.py b/tests/test_cache.py index b7d62e9..2866723 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -378,6 +378,68 @@ def wrapper(raw: Any) -> Any: cache_mod.cache_strategy("global") +def test_set_cache_limit_zero_clears_cache(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(cache_mod._pcre2, "compile", lambda pattern, *, flags=0, jit=False: pattern) + + def wrapper(raw: Any) -> Any: + return raw + + store = _fresh_thread_cache() + cache_mod.cached_compile("expr", 0, wrapper, jit=False) + assert len(store) == 1 + + cache_mod.set_cache_limit(0) + assert cache_mod.get_cache_limit() == 0 + assert len(store) == 0 + + # With the limit set to 0, compilation should not be cached. + cache_mod.cached_compile("expr", 0, wrapper, jit=False) + assert len(store) == 0 + + +def test_set_cache_limit_shrink_evicts_oldest(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(cache_mod._pcre2, "compile", lambda pattern, *, flags=0, jit=False: pattern) + + def wrapper(raw: Any) -> Any: + return raw + + store = _fresh_thread_cache() + cache_mod.set_cache_limit(10) + for i in range(5): + cache_mod.cached_compile(str(i), 0, wrapper, jit=False) + assert len(store) == 5 + + cache_mod.set_cache_limit(2) + assert len(store) == 2 + assert list(store.keys()) == [("3", 0, False), ("4", 0, False)] + + +def test_cached_compile_bypasses_cache_for_unhashable_key(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[Any] = [] + + def fake_compile(pattern: Any, *, flags: int = 0, jit: bool = False) -> Any: + calls.append(pattern) + return pattern + + monkeypatch.setattr(cache_mod._pcre2, "compile", fake_compile) + + def wrapper(raw: Any) -> Any: + return raw + + store = _fresh_thread_cache() + # A list is not hashable, so the cache lookup must fall back to compiling each time. + first = cache_mod.cached_compile(["unhashable"], 0, wrapper, jit=False) + second = cache_mod.cached_compile(["unhashable"], 0, wrapper, jit=False) + assert first == second + assert len(store) == 0 + assert calls == [["unhashable"], ["unhashable"]] + + +def test_set_cache_limit_rejects_negative() -> None: + with pytest.raises(ValueError): + cache_mod.set_cache_limit(-1) + + def test_thread_cache_cleanup_no_leak() -> None: script = textwrap.dedent( """ diff --git a/tests/test_cache_global.py b/tests/test_cache_global.py new file mode 100644 index 0000000..8572f1f --- /dev/null +++ b/tests/test_cache_global.py @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: 2025 ModelCloud.ai +# SPDX-FileCopyrightText: 2025 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +"""Coverage tests for the global pattern-cache strategy.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap +from pathlib import Path +from typing import Any, Dict + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + + +def _run_global_cache_script(source: str) -> Dict[str, Any]: + env = os.environ.copy() + pythonpath_entries = [str(PROJECT_ROOT)] + existing_pythonpath = env.get("PYTHONPATH") + if existing_pythonpath: + pythonpath_entries.append(existing_pythonpath) + env["PYTHONPATH"] = os.pathsep.join(pythonpath_entries) + env["PYPCRE_CACHE_PATTERN_GLOBAL"] = "1" + + completed = subprocess.run( + [sys.executable, "-c", source], + check=True, + capture_output=True, + text=True, + env=env, + ) + if completed.stderr: + raise AssertionError(f"unexpected stderr output: {completed.stderr}") + return json.loads(completed.stdout) + + +def test_global_cache_strategy_is_active() -> None: + script = textwrap.dedent( + """ + import pcre.cache as cache_mod + import json + print(json.dumps({"strategy": cache_mod.cache_strategy()})) + """ + ) + result = _run_global_cache_script(script) + assert result["strategy"] == "global" + + +def test_global_cache_limit_and_clear() -> None: + script = textwrap.dedent( + """ + import pcre.cache as cache_mod + import json + + def wrapper(raw): + return raw + + cache_mod.clear_cache() + cache_mod.set_cache_limit(2) + cache_mod.cached_compile("a", 0, wrapper, jit=False) + cache_mod.cached_compile("b", 0, wrapper, jit=False) + cache_mod.cached_compile("c", 0, wrapper, jit=False) + + before = len(cache_mod._GLOBAL_STATE.pattern_cache) + cache_mod.clear_cache() + after = len(cache_mod._GLOBAL_STATE.pattern_cache) + limit = cache_mod.get_cache_limit() + print(json.dumps({"before": before, "after": after, "limit": limit})) + """ + ) + result = _run_global_cache_script(script) + assert result["before"] == 2 + assert result["after"] == 0 + assert result["limit"] == 2 + + +def test_global_cache_limit_zero_disables_caching() -> None: + script = textwrap.dedent( + """ + import pcre.cache as cache_mod + import json + + calls = [] + def wrapper(raw): + calls.append(raw) + return raw + + def fake_compile(pattern, *, flags=0, jit=False): + calls.append(("compile", pattern)) + return pattern + + cache_mod._pcre2.compile = fake_compile + cache_mod.clear_cache() + cache_mod.set_cache_limit(0) + + cache_mod.cached_compile("x", 0, wrapper, jit=False) + cache_mod.cached_compile("x", 0, wrapper, jit=False) + + print(json.dumps({"calls": calls})) + """ + ) + result = _run_global_cache_script(script) + assert result["calls"].count(["compile", "x"]) == 2 + + +def test_global_cache_shrinks_to_new_limit() -> None: + script = textwrap.dedent( + """ + import pcre.cache as cache_mod + import json + + def wrapper(raw): + return raw + + def fake_compile(pattern, *, flags=0, jit=False): + return pattern + + cache_mod._pcre2.compile = fake_compile + cache_mod.clear_cache() + cache_mod.set_cache_limit(10) + for i in range(5): + cache_mod.cached_compile(str(i), 0, wrapper, jit=False) + + cache_mod.set_cache_limit(2) + size = len(cache_mod._GLOBAL_STATE.pattern_cache) + print(json.dumps({"size": size})) + """ + ) + result = _run_global_cache_script(script) + assert result["size"] == 2 diff --git a/tests/test_cache_global_inproc.py b/tests/test_cache_global_inproc.py new file mode 100644 index 0000000..a9bd714 --- /dev/null +++ b/tests/test_cache_global_inproc.py @@ -0,0 +1,163 @@ +# SPDX-FileCopyrightText: 2025 ModelCloud.ai +# SPDX-FileCopyrightText: 2025 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +"""In-process coverage tests for the global pattern-cache strategy.""" + +from __future__ import annotations + +from typing import Any + +import pcre.cache as cache_mod +import pytest + + +def test_env_flag_is_true() -> None: + assert cache_mod._env_flag_is_true(None) is False + assert cache_mod._env_flag_is_true("") is False + assert cache_mod._env_flag_is_true("0") is False + assert cache_mod._env_flag_is_true("false") is False + assert cache_mod._env_flag_is_true("FALSE") is False + assert cache_mod._env_flag_is_true("no") is False + assert cache_mod._env_flag_is_true("NO") is False + assert cache_mod._env_flag_is_true("1") is True + assert cache_mod._env_flag_is_true("true") is True + + +def test_global_cached_compile(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[Any] = [] + + def fake_compile(pattern: Any, *, flags: int = 0, jit: bool = False) -> Any: + calls.append((pattern, flags, jit)) + return pattern + + monkeypatch.setattr(cache_mod._pcre2, "compile", fake_compile) + + fresh_state = cache_mod._GlobalCacheState() + monkeypatch.setattr(cache_mod, "_GLOBAL_STATE", fresh_state) + monkeypatch.setattr(cache_mod, "_CACHE_STRATEGY", cache_mod._CacheStrategy.GLOBAL) + + def wrapper(raw: Any) -> Any: + return f"wrapped:{raw}" + + first = cache_mod.cached_compile("expr", 0, wrapper, jit=False) + assert first == "wrapped:expr" + assert len(calls) == 1 + + second = cache_mod.cached_compile("expr", 0, wrapper, jit=False) + assert second == first + assert len(calls) == 1 + + +def test_global_cache_clear_and_limits(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[Any] = [] + + def fake_compile(pattern: Any, *, flags: int = 0, jit: bool = False) -> Any: + calls.append(pattern) + return pattern + + monkeypatch.setattr(cache_mod._pcre2, "compile", fake_compile) + + fresh_state = cache_mod._GlobalCacheState() + monkeypatch.setattr(cache_mod, "_GLOBAL_STATE", fresh_state) + monkeypatch.setattr(cache_mod, "_CACHE_STRATEGY", cache_mod._CacheStrategy.GLOBAL) + + def wrapper(raw: Any) -> Any: + return raw + + cache_mod.cached_compile("a", 0, wrapper, jit=False) + cache_mod.cached_compile("b", 0, wrapper, jit=False) + assert len(fresh_state.pattern_cache) == 2 + + cache_mod.clear_cache() + assert len(fresh_state.pattern_cache) == 0 + + cache_mod.set_cache_limit(1) + assert cache_mod.get_cache_limit() == 1 + cache_mod.cached_compile("x", 0, wrapper, jit=False) + cache_mod.cached_compile("y", 0, wrapper, jit=False) + assert len(fresh_state.pattern_cache) == 1 + + cache_mod.set_cache_limit(0) + assert cache_mod.get_cache_limit() == 0 + assert len(fresh_state.pattern_cache) == 0 + + cache_mod.cached_compile("z", 0, wrapper, jit=False) + assert len(fresh_state.pattern_cache) == 0 + + +def test_global_set_cache_limit_none(monkeypatch: pytest.MonkeyPatch) -> None: + fresh_state = cache_mod._GlobalCacheState() + monkeypatch.setattr(cache_mod, "_GLOBAL_STATE", fresh_state) + monkeypatch.setattr(cache_mod, "_CACHE_STRATEGY", cache_mod._CacheStrategy.GLOBAL) + + cache_mod.set_cache_limit(None) + assert cache_mod.get_cache_limit() is None + + +def test_global_set_cache_limit_shrinks_existing_entries(monkeypatch: pytest.MonkeyPatch) -> None: + def fake_compile(pattern: Any, *, flags: int = 0, jit: bool = False) -> Any: + return pattern + + monkeypatch.setattr(cache_mod._pcre2, "compile", fake_compile) + + fresh_state = cache_mod._GlobalCacheState() + monkeypatch.setattr(cache_mod, "_GLOBAL_STATE", fresh_state) + monkeypatch.setattr(cache_mod, "_CACHE_STRATEGY", cache_mod._CacheStrategy.GLOBAL) + + def wrapper(raw: Any) -> Any: + return raw + + cache_mod.cached_compile("a", 0, wrapper, jit=False) + cache_mod.cached_compile("b", 0, wrapper, jit=False) + assert len(fresh_state.pattern_cache) == 2 + + cache_mod.set_cache_limit(1) + assert len(fresh_state.pattern_cache) == 1 + + +def test_global_cached_compile_handles_unhashable_key(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[Any] = [] + + def fake_compile(pattern: Any, *, flags: int = 0, jit: bool = False) -> Any: + calls.append(pattern) + return pattern + + monkeypatch.setattr(cache_mod._pcre2, "compile", fake_compile) + + fresh_state = cache_mod._GlobalCacheState() + monkeypatch.setattr(cache_mod, "_GLOBAL_STATE", fresh_state) + monkeypatch.setattr(cache_mod, "_CACHE_STRATEGY", cache_mod._CacheStrategy.GLOBAL) + + def wrapper(raw: Any) -> Any: + return raw + + first = cache_mod.cached_compile(["unhashable"], 0, wrapper, jit=False) + second = cache_mod.cached_compile(["unhashable"], 0, wrapper, jit=False) + assert first == second + assert len(calls) == 2 + assert len(fresh_state.pattern_cache) == 0 + + +def test_global_cached_compile_respects_limit_set_to_zero_after_compile(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[Any] = [] + + def fake_compile(pattern: Any, *, flags: int = 0, jit: bool = False) -> Any: + calls.append(pattern) + return pattern + + monkeypatch.setattr(cache_mod._pcre2, "compile", fake_compile) + + fresh_state = cache_mod._GlobalCacheState() + monkeypatch.setattr(cache_mod, "_GLOBAL_STATE", fresh_state) + monkeypatch.setattr(cache_mod, "_CACHE_STRATEGY", cache_mod._CacheStrategy.GLOBAL) + + def wrapper(raw: Any) -> Any: + # Simulate another thread setting the limit to zero between the compile + # and the cache insertion. + fresh_state.cache_limit = 0 + return raw + + cache_mod.cached_compile("x", 0, wrapper, jit=False) + assert len(fresh_state.pattern_cache) == 0 diff --git a/tests/test_coverage_gaps.py b/tests/test_coverage_gaps.py new file mode 100644 index 0000000..2d67fa2 --- /dev/null +++ b/tests/test_coverage_gaps.py @@ -0,0 +1,366 @@ +# SPDX-FileCopyrightText: 2025 ModelCloud.ai +# SPDX-FileCopyrightText: 2025 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +"""Coverage tests for pcre.pcre corner cases missing from the main suites.""" + +from __future__ import annotations + +from typing import Any + +import pcre +import pcre.pcre as pcre_mod +import pytest + + +def test_compile_rejects_invalid_flags_type() -> None: + with pytest.raises(TypeError): + pcre.compile("a", flags="foo") + with pytest.raises(TypeError): + pcre.compile("a", flags=b"foo") + with pytest.raises(TypeError): + pcre.compile("a", flags=1.5) + + +def test_compile_rejects_conflicting_thread_flags() -> None: + with pytest.raises(ValueError, match="cannot be combined"): + pcre.compile("a", flags=pcre.Flag.THREADS | pcre.Flag.NO_THREADS) + + +def test_compile_pattern_instance_with_thread_flags() -> None: + pattern = pcre.compile("a") + + pcre.compile(pattern, flags=pcre.Flag.THREADS) + assert pattern.use_threads is True + + pcre.compile(pattern, flags=pcre.Flag.NO_THREADS) + assert pattern.use_threads is False + + +def test_compile_pattern_instance_with_compat_flag_raises() -> None: + pattern = pcre.compile("a") + with pytest.raises(ValueError, match="COMPAT_UNICODE_ESCAPE"): + pcre.compile(pattern, flags=pcre.Flag.COMPAT_UNICODE_ESCAPE) + + +def test_compile_pattern_instance_with_other_flags_raises() -> None: + pattern = pcre.compile("a") + with pytest.raises(ValueError, match="Cannot supply flags"): + pcre.compile(pattern, flags=pcre.Flag.MULTILINE) + + +def test_compile_compiled_c_pattern_with_thread_flags(monkeypatch: pytest.MonkeyPatch) -> None: + raw = pcre.compile("a")._pattern + + pcre.compile(raw, flags=pcre.Flag.THREADS) + pcre.compile(raw, flags=pcre.Flag.NO_THREADS) + + with pytest.raises(ValueError, match="Cannot supply flags"): + pcre.compile(raw, flags=pcre.Flag.MULTILINE) + + with pytest.raises(ValueError, match="COMPAT_UNICODE_ESCAPE"): + pcre.compile(raw, flags=pcre.Flag.COMPAT_UNICODE_ESCAPE) + + +def test_compile_respects_disabled_thread_default(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pcre_mod, "get_thread_default", lambda: False) + + # Default thread mode disabled for string patterns. + pattern = pcre.compile("a") + assert pattern.thread_mode == pcre_mod._THREAD_MODE_DISABLED + + # Explicit thread flag still works. + pattern = pcre.compile("a", flags=pcre.Flag.THREADS) + assert pattern.thread_mode == pcre_mod._THREAD_MODE_ENABLED + + raw = pcre.compile("b")._pattern + pattern = pcre.compile(raw) + assert pattern.thread_mode == pcre_mod._THREAD_MODE_DISABLED + pattern = pcre.compile(raw, flags=pcre.Flag.NO_THREADS) + assert pattern.thread_mode == pcre_mod._THREAD_MODE_DISABLED + + +def test_compile_respects_disabled_thread_default_with_no_threads(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pcre_mod, "get_thread_default", lambda: False) + pattern = pcre.compile("a", flags=pcre.Flag.NO_THREADS) + assert pattern.thread_mode == pcre_mod._THREAD_MODE_DISABLED + + +def test_match_methods_accept_memoryview() -> None: + pattern = pcre.compile(br"\w+") + data = memoryview(b"hello world") + + assert pattern.match(data).group(0) == b"hello" + assert pattern.match(data, endpos=4).group(0) == b"hell" + assert pattern.match(data, endpos=0) is None + + assert pattern.search(data).group(0) == b"hello" + assert pattern.search(data, pos=6).group(0) == b"world" + assert pattern.search(data, pos=6, endpos=7).group(0) == b"w" + assert pattern.search(data, pos=6, endpos=6) is None + + full_pattern = pcre.compile(br"hello") + assert full_pattern.fullmatch(data, endpos=5).group(0) == b"hello" + assert full_pattern.fullmatch(data, endpos=4) is None + assert full_pattern.fullmatch(data, endpos=0) is None + + assert pattern.findall(data) == [b"hello", b"world"] + assert list(pattern.finditer(data)) != [] + + +def test_no_match_with_memoryview_and_endpos() -> None: + pattern = pcre.compile(br"\d+") + data = memoryview(b"hello world") + + assert pattern.match(data) is None + assert pattern.match(data, endpos=0) is None + assert pattern.search(data) is None + assert pattern.fullmatch(data) is None + assert pattern.findall(data) == [] + assert list(pattern.finditer(data)) == [] + + +def test_compat_match_path_used_when_attach_unavailable(monkeypatch: pytest.MonkeyPatch) -> None: + """Force the pure-Python Match wrapper path used by older backends.""" + + monkeypatch.setattr(pcre_mod, "_can_attach_match", lambda raw: False) + + pattern = pcre.compile(r"(a)(b)?") + match = pattern.search("xay") + assert match.group(0) == "a" + assert match.group(1) == "a" + assert match.group(2) is None + assert match.groups() == ("a", None) + assert match.start() == 1 + assert match.end() == 2 + assert match.span() == (1, 2) + assert match.re is pattern + assert match.string == "xay" + assert match.pos == 0 + assert match.lastindex == 1 + assert match.lastgroup is None + + +def test_finditer_zero_length_pattern_advances() -> None: + matches = list(pcre.finditer(r"", "abc")) + assert len(matches) == 4 + + +def test_module_template_function() -> None: + with pytest.warns(DeprecationWarning): + pattern = pcre.template("(a)") + assert pattern.search("a").group(0) == "a" + + +def test_parallel_map_empty_subjects() -> None: + pattern = pcre.compile(r"\d+", flags=pcre.Flag.THREADS) + assert pcre.parallel_map(pattern, []) == [] + + +def test_parallel_map_invalid_method() -> None: + pattern = pcre.compile(r"\d+", flags=pcre.Flag.THREADS) + with pytest.raises(ValueError, match="parallel_map"): + pcre.parallel_map(pattern, ["1"], method="split") + + +def test_parallel_map_auto_threads_with_small_subjects(monkeypatch: pytest.MonkeyPatch) -> None: + """When auto threshold is high, small subjects are processed sequentially.""" + monkeypatch.setattr(pcre_mod, "get_auto_threshold", lambda: 1_000_000) + pattern = pcre.compile(r"\d+", flags=pcre.Flag.THREADS) + results = pcre.parallel_map(pattern, ["1", "22", "333"]) + assert [m.group(0) for m in results] == ["1", "22", "333"] + + +def test_parallel_map_low_threshold_uses_threads(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pcre_mod, "get_auto_threshold", lambda: 0) + pattern = pcre.compile(r"\d+", flags=pcre.Flag.THREADS) + results = pcre.parallel_map(pattern, ["1", "22", "333"]) + assert [m.group(0) for m in results] == ["1", "22", "333"] + + +def test_parallel_map_requires_thread_enabled() -> None: + pattern = pcre.compile(r"\d+", flags=pcre.Flag.NO_THREADS) + with pytest.raises(RuntimeError, match="not enabled for threaded execution"): + pcre.parallel_map(pattern, ["1"]) + + +def test_parallel_map_memoryview_subject(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pcre_mod, "get_auto_threshold", lambda: 0) + pattern = pcre.compile(br"\d+", flags=pcre.Flag.THREADS) + results = pcre.parallel_map(pattern, [memoryview(b"1"), memoryview(b"22")]) + assert [m.group(0) for m in results] == [b"1", b"22"] + + +def test_parallel_map_threading_unsupported_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pcre_mod, "threading_supported", lambda: False) + pattern = pcre.compile(r"\d+", flags=pcre.Flag.THREADS) + results = pcre.parallel_map(pattern, ["1", "22"]) + assert [m.group(0) for m in results] == ["1", "22"] + + +def test_compile_disabled_default_with_non_thread_flags(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pcre_mod, "get_thread_default", lambda: False) + pattern = pcre.compile("a", flags=pcre.Flag.MULTILINE) + assert pattern.thread_mode == pcre_mod._THREAD_MODE_DISABLED + + +def test_parallel_map_auto_mode_uses_threshold_zero(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pcre_mod, "get_auto_threshold", lambda: 0) + pattern = pcre.compile(r"\d+") + results = pcre.parallel_map(pattern, ["1", "22", "333"]) + assert [m.group(0) for m in results] == ["1", "22", "333"] + + +def test_parallel_map_auto_mode_with_memoryview_subject(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pcre_mod, "get_auto_threshold", lambda: 1_000_000) + pattern = pcre.compile(br"\d+") + results = pcre.parallel_map(pattern, [memoryview(b"1"), memoryview(b"22")]) + assert [m.group(0) for m in results] == [b"1", b"22"] + + +def test_pattern_parallel_map_requires_thread_enabled() -> None: + pattern = pcre.compile(r"\d+", flags=pcre.Flag.NO_THREADS) + with pytest.raises(RuntimeError, match="not enabled for threaded execution"): + pattern.parallel_map(["1"]) + + +class _FakeRawMatch: + def __init__(self, value: str, span: tuple[int, int], groups: tuple[str, ...] = ()) -> None: + self._value = value + self._span = span + self._groups = groups + + def group(self, *indices: int) -> Any: + if not indices: + return self._value + if len(indices) == 1: + if indices[0] == 0: + return self._value + idx = indices[0] - 1 + return self._groups[idx] if 0 <= idx < len(self._groups) else None + return tuple(self.group(i) for i in indices) + + def groups(self, default: Any = None) -> tuple[Any, ...]: + return self._groups + + def span(self, group: int = 0) -> tuple[int, int]: + return self._span + + def start(self, group: int = 0) -> int: + return self._span[0] + + def end(self, group: int = 0) -> int: + return self._span[1] + + +class _FakeCPattern: + def __init__(self, *, findall: Any = None, finditer: Any = None) -> None: + self.pattern = r"(a)(b?)" + self.groupindex = {} + self.flags = 0 + self.capture_count = 2 + self.jit = False + self._findall = findall + self._finditer = finditer + self._call_count = 0 + + def match(self, subject: Any, *, pos: int = 0, endpos: int = -1, options: int = 0) -> Any: + if pos == 0 and subject.startswith("ab", pos): + return _FakeRawMatch("ab", (0, 2), ("a", "b")) + return None + + def search(self, subject: Any, *, pos: int = 0, endpos: int = -1, options: int = 0) -> Any: + subject_str = subject.decode() if isinstance(subject, bytes) else subject + length = len(subject_str) + if pos < length: + return _FakeRawMatch(subject_str[pos:pos + 1], (pos, pos + 1), (subject_str[pos:pos + 1],)) + if pos == length: + # Zero-width match at the end, like re.finditer(''). + return _FakeRawMatch("", (pos, pos), ()) + return None + + def fullmatch(self, subject: Any, *, pos: int = 0, endpos: int = -1, options: int = 0) -> Any: + if endpos == -1: + endpos = len(subject) + if pos == 0 and endpos == 1 and (subject == "a" or subject == b"a"): + return _FakeRawMatch("a", (0, 1), ("a",)) + return None + + def findall(self, subject: Any, *, pos: int = 0, endpos: int = -1, options: int = 0) -> Any: + if self._findall is not None: + return self._findall(subject, pos=pos, endpos=endpos, options=options) + raise TypeError("findall not implemented") + + def finditer(self, subject: Any, *, pos: int = 0, endpos: int = -1, options: int = 0) -> Any: + if self._finditer is not None: + return self._finditer(subject, pos=pos, endpos=endpos, options=options) + raise TypeError("finditer not implemented") + + +def _fake_pattern(**kwargs: Any) -> pcre_mod.Pattern: + return pcre_mod.Pattern(_FakeCPattern(**kwargs)) + + +def test_finditer_uses_backend_iterator() -> None: + raw_matches = [ + _FakeRawMatch("a", (0, 1), ("a",)), + _FakeRawMatch("b", (1, 2), ("b",)), + ] + pattern = _fake_pattern(finditer=lambda subject, **kw: iter(raw_matches)) + matches = list(pattern.finditer("ab")) + assert [m.group(0) for m in matches] == ["a", "b"] + + +def test_findall_uses_backend_iterator() -> None: + raw_matches = [ + _FakeRawMatch("a", (0, 1), ("a",)), + _FakeRawMatch("b", (1, 2), ("b",)), + _FakeRawMatch("", (2, 2), ()), + ] + pattern = _fake_pattern(finditer=lambda subject, **kw: iter(raw_matches)) + assert pattern.findall("ab") == ["a", "b", ""] + + +def test_finditer_fallback_loop_advances_zero_width_matches() -> None: + pattern = _fake_pattern() + matches = list(pattern.finditer("ab")) + assert len(matches) == 3 # positions 0, 1, and a zero-width end match. + + +def test_finditer_fallback_loop_handles_zero_width_at_endpos() -> None: + pattern = _fake_pattern() + matches = list(pattern.finditer("ab", pos=2, endpos=2)) + assert len(matches) == 1 + + +def test_findall_fallback_loop() -> None: + pattern = _fake_pattern() + assert pattern.findall("ab") == ["a", "b", ""] + + +def test_fullmatch_compat_match_path(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pcre_mod, "_can_attach_match", lambda raw: False) + pattern = pcre.compile(r"a") + assert pattern.fullmatch("a").group(0) == "a" + assert pattern.fullmatch("a", endpos=0) is None + + +def test_sub_renders_template_when_group_hint_missing() -> None: + pattern = _fake_pattern() + pattern._groups_hint = None + result = pattern.sub(r"[\1]", "ab", count=2) + assert result == "[a][b]" + assert pattern._groups_hint == 1 + + +def test_sub_raises_pcre_error_for_invalid_template() -> None: + pattern = _fake_pattern() + pattern._groups_hint = None + with pytest.raises(pcre.PcreError): + pattern.sub(r"[\2]", "ab") + + +def test_wrap_match_returns_none_for_none_raw() -> None: + pattern = _fake_pattern() + assert pattern._wrap_match(None, "x", 0, 1) is None diff --git a/tests/test_errors.py b/tests/test_errors.py index ebfaadf..55fd8dc 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -75,6 +75,12 @@ def test_low_level_compile_reports_bad_options(): assert exc.error_code is pcre.PcreErrorCode.BAD_OPTIONS +def test_error_code_property_returns_none_for_unknown_code(): + exc = pcre.PcreErrorBadOptions("test") + exc.code = 99999 + assert exc.error_code is None + + def test_substitution_with_invalid_numeric_group_reference_raises(): # Regression test: patterns that only expose numeric group names through # DUPNAMES should still reject ``\1`` substitutions when no real capture diff --git a/tests/test_re_compat.py b/tests/test_re_compat.py new file mode 100644 index 0000000..5d6436f --- /dev/null +++ b/tests/test_re_compat.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: 2025 ModelCloud.ai +# SPDX-FileCopyrightText: 2025 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +"""Coverage tests for the public helpers in pcre.re_compat.""" + +from __future__ import annotations + +from typing import Any + +import pcre.re_compat as rc +import pytest + + +def test_prepare_subject_memoryview() -> None: + data = b"hello" + assert rc.prepare_subject(memoryview(data)) == data + assert rc.prepare_subject("hello") == "hello" + + +def test_is_bytes_like() -> None: + assert rc.is_bytes_like(b"x") is True + assert rc.is_bytes_like(bytearray(b"x")) is True + assert rc.is_bytes_like(memoryview(b"x")) is True + assert rc.is_bytes_like("x") is False + + +def test_normalise_count() -> None: + assert rc.normalise_count(3) == 3 + assert rc.normalise_count(0) is None + assert rc.normalise_count(-1) is None + assert rc.normalise_count(None) is None + assert rc.normalise_count(True) == 1 + + +def test_resolve_endpos() -> None: + assert rc.resolve_endpos("hello", None) == 5 + assert rc.resolve_endpos("hello", -1) == 5 + assert rc.resolve_endpos("hello", 3) == 3 + + +def test_compute_next_pos() -> None: + assert rc.compute_next_pos(0, (1, 3), None) == 3 + assert rc.compute_next_pos(0, (2, 2), None) == 3 + assert rc.compute_next_pos(0, (2, 5), 4) == 5 + assert rc.compute_next_pos(0, (2, 5), 3) == 5 + + +def test_coerce_group_value() -> None: + assert rc.coerce_group_value("x", is_bytes=False, empty="") == "x" + assert rc.coerce_group_value(None, is_bytes=False, empty="") == "" + assert rc.coerce_group_value(b"x", is_bytes=True, empty=b"") == b"x" + assert rc.coerce_group_value(bytearray(b"x"), is_bytes=True, empty=b"") == b"x" + assert rc.coerce_group_value(memoryview(b"x"), is_bytes=True, empty=b"") == b"x" + with pytest.raises(TypeError): + rc.coerce_group_value("x", is_bytes=True, empty=b"") + with pytest.raises(TypeError): + rc.coerce_group_value(b"x", is_bytes=False, empty="") + + +def test_coerce_subject_slice() -> None: + assert rc.coerce_subject_slice("hello", 1, 3, is_bytes=False) == "el" + assert rc.coerce_subject_slice(b"hello", 1, 3, is_bytes=True) == b"el" + assert rc.coerce_subject_slice(bytearray(b"hello"), 1, 3, is_bytes=True) == b"el" + assert rc.coerce_subject_slice(memoryview(b"hello"), 1, 3, is_bytes=True) == b"el" + + +def test_normalise_replacement() -> None: + assert rc.normalise_replacement("x", is_bytes=False) == "x" + assert rc.normalise_replacement(b"x", is_bytes=True) == b"x" + assert rc.normalise_replacement(bytearray(b"x"), is_bytes=True) == b"x" + assert rc.normalise_replacement(memoryview(b"x"), is_bytes=True) == b"x" + with pytest.raises(TypeError): + rc.normalise_replacement("x", is_bytes=True) + with pytest.raises(TypeError): + rc.normalise_replacement(b"x", is_bytes=False) + + +def test_join_parts() -> None: + assert rc.join_parts(["a", "b"], is_bytes=False) == "ab" + assert rc.join_parts([b"a", b"b"], is_bytes=True) == b"ab" + assert rc.join_parts([bytearray(b"a"), memoryview(b"b")], is_bytes=True) == b"ab" + with pytest.raises(TypeError): + rc.join_parts(["a"], is_bytes=True) + + +def test_render_template_tuple_format() -> None: + """Python 3.11 returns (group_slots, literals); newer versions return a flat list.""" + + class FakeMatch: + def group(self, index: int) -> str: + return {1: "X", 2: "YY"}.get(index, "") + + match = FakeMatch() + # Tuple format: group_slots=[(1, 1)], literals=["pre", None, "post"] + assert rc.render_template(([(1, 1)], ["pre", None, "post"]), match, is_bytes=False, empty="") == "preXpost" + # Flat list format with group reference ints and literal strings. + assert rc.render_template(["pre", 1, "post"], match, is_bytes=False, empty="") == "preXpost" + + +def test_expand_match_template_bytes() -> None: + import pcre + + pattern = pcre.compile(br"(\w+)") + match = pattern.search(b"hello world") + assert match.expand(br"[\1]") == b"[hello]" + + +def test_count_capturing_groups_variants() -> None: + assert rc.count_capturing_groups("(a)(b)") == 2 + assert rc.count_capturing_groups(b"(a)(b)") == 2 + assert rc.count_capturing_groups(bytearray(b"(a)(b)")) == 2 + assert rc.count_capturing_groups(memoryview(b"(a)(b)")) == 2 + # Character classes and escapes should not be counted as captures. + assert rc.count_capturing_groups(r"[a(b)]") == 0 + assert rc.count_capturing_groups(r"a\(b") == 0 + assert rc.count_capturing_groups(r"(?Pa)(?:(?<=x)y)(a)") == 2 + assert rc.count_capturing_groups(r"(?>a)(?:b)(?=c)(?!d)(?<=e)(?name)") == 0 + assert rc.count_capturing_groups(r"(?Pa)(?P'name')(?b)(?'name3'c)") == 4 + + +def test_is_capturing_group_start() -> None: + assert rc.is_capturing_group_start("(a)", 0) is True + assert rc.is_capturing_group_start("(?:a)", 0) is False + assert rc.is_capturing_group_start("(?Pa)", 0) is True + assert rc.is_capturing_group_start("(?P=n)", 0) is False + assert rc.is_capturing_group_start("(?P>n)", 0) is False + assert rc.is_capturing_group_start("(?|a|b)", 0) is True + assert rc.is_capturing_group_start("(?=a)", 0) is False + assert rc.is_capturing_group_start("(?!a)", 0) is False + assert rc.is_capturing_group_start("(?<=a)", 0) is False + assert rc.is_capturing_group_start("(?a)", 0) is True + assert rc.is_capturing_group_start("(?<=a)", 0) is False + assert rc.is_capturing_group_start("(? None: + assert rc.maybe_infer_group_count(r"(a)(b)") == 2 + assert rc.maybe_infer_group_count(b"(a)(b)") == 2 + assert rc.maybe_infer_group_count(bytearray(b"(a)(b)")) == 2 + assert rc.maybe_infer_group_count(memoryview(b"(a)(b)")) == 2 + + +class FakeMatch: + def __init__(self, groups: tuple[Any, ...], pattern: Any, string: Any) -> None: + self._groups = groups + self.re = pattern + self.string = string + + def group(self, *indices: Any) -> Any: + if len(indices) == 1: + return self._groups[indices[0]] + return tuple(self._groups[i] for i in indices) + + def groups(self, default: Any = None) -> tuple[Any, ...]: + return self._groups[1:] + + def groupdict(self, default: Any = None) -> dict[str, Any]: + return {} + + def start(self, group: int = 0) -> int: + return 0 + + def end(self, group: int = 0) -> int: + return 0 + + def span(self, group: int = 0) -> tuple[int, int]: + return (0, 0) + + def expand(self, template: str) -> str: + return rc.expand_match_template(self, template) + + +def test_re_compat_match_object() -> None: + pattern = type("Pattern", (), {"groupindex": {"word": 1}, "groups": 1})() + match = rc.Match(pattern, FakeMatch(("full", "value"), pattern, "value"), "value", 0, 5) + assert match.group(0) == "full" + assert match.groups() == ("value",) + assert match.groupdict() == {} + assert match.start() == 0 + assert match.end() == 0 + assert match.span() == (0, 0) + assert match[0] == "full" + assert match.lastindex == 1 + assert match.lastgroup == "word" + assert match.regs == ((0, 0), (0, 0)) + assert match.string == "value" + assert match.pos == 0 + assert match.endpos == 5 + + +def test_re_compat_match_lastindex_none() -> None: + pattern = type("Pattern", (), {"groupindex": {}, "groups": 0})() + match = rc.Match(pattern, FakeMatch(("full",), pattern, "x"), "x", 0, 1) + assert match.lastindex is None + assert match.lastgroup is None + + +def test_match_expand() -> None: + pattern = type("Pattern", (), {"groupindex": {"word": 1}, "groups": 1})() + match = rc.Match(pattern, FakeMatch(("full", "value"), pattern, "value"), "value", 0, 5) + assert match.expand(r"[\1]") == "[value]" + + +def test_expand_match_template_type_errors() -> None: + pattern = type("Pattern", (), {"groupindex": {}, "groups": 1})() + text_match = rc.Match(pattern, FakeMatch(("hello", "world"), pattern, "hello world"), "hello world", 0, 11) + bytes_match = rc.Match(pattern, FakeMatch((b"hello", b"world"), pattern, b"hello world"), b"hello world", 0, 11) + + with pytest.raises(TypeError): + rc.expand_match_template(text_match, b"[\1]") + with pytest.raises(TypeError): + rc.expand_match_template(bytes_match, "[\\1]") + + diff --git a/tests/test_threads.py b/tests/test_threads.py new file mode 100644 index 0000000..bb10481 --- /dev/null +++ b/tests/test_threads.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: 2025 ModelCloud.ai +# SPDX-FileCopyrightText: 2025 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +"""Coverage tests for pcre.threads configuration helpers.""" + +from __future__ import annotations + +import pcre.threads as threads_mod +import pytest + + +def test_threading_supported_false_on_low_core_count(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(threads_mod, "_cpu_total", lambda: 4) + assert threads_mod.threading_supported() is False + + +def test_configure_threads_toggles_default(monkeypatch: pytest.MonkeyPatch) -> None: + original = threads_mod.get_thread_default() + try: + assert threads_mod.configure_threads(enabled=True) is True + assert threads_mod.get_thread_default() is True + assert threads_mod.configure_threads(enabled=False) is False + assert threads_mod.get_thread_default() is False + finally: + threads_mod.configure_threads(enabled=original) + + +def test_configure_threads_threshold_validation(monkeypatch: pytest.MonkeyPatch) -> None: + original = threads_mod.get_auto_threshold() + try: + assert threads_mod.configure_threads(threshold=1234) == threads_mod.get_thread_default() + assert threads_mod.get_auto_threshold() == 1234 + + with pytest.raises(ValueError): + threads_mod.configure_threads(threshold=-1) + with pytest.raises(TypeError): + threads_mod.configure_threads(threshold="not-an-int") + finally: + threads_mod.configure_threads(threshold=original) + + +def test_configure_thread_pool_clamps_workers(monkeypatch: pytest.MonkeyPatch) -> None: + original_workers = getattr(threads_mod, "_THREAD_POOL_WORKERS", None) + original_pool = getattr(threads_mod, "_THREAD_POOL", None) + try: + # Use a huge worker request; it should be clamped to the computed maximum. + maximum = threads_mod._max_threads() + workers = threads_mod.configure_thread_pool(max_workers=10000, preload=False) + assert workers == maximum + + # Preload creates the pool eagerly. + threads_mod.shutdown_thread_pool(wait=True) + workers = threads_mod.configure_thread_pool(max_workers=2, preload=True) + assert workers == max(1, min(2, maximum)) + assert threads_mod._THREAD_POOL is not None + finally: + threads_mod.shutdown_thread_pool(wait=True) + threads_mod._THREAD_POOL = original_pool + threads_mod._THREAD_POOL_WORKERS = original_workers + + +def test_configure_thread_pool_rejects_invalid_worker_count() -> None: + with pytest.raises(TypeError): + threads_mod.configure_thread_pool(max_workers="x") + with pytest.raises(ValueError): + threads_mod.configure_thread_pool(max_workers=0) + + +def test_get_thread_pool_size_initializes_once(monkeypatch: pytest.MonkeyPatch) -> None: + original_workers = threads_mod._THREAD_POOL_WORKERS + original_pool = threads_mod._THREAD_POOL + try: + threads_mod._THREAD_POOL_WORKERS = None + threads_mod._THREAD_POOL = None + size = threads_mod.get_thread_pool_size() + assert size > 0 + # Second call returns the cached value. + assert threads_mod.get_thread_pool_size() == size + finally: + threads_mod._THREAD_POOL_WORKERS = original_workers + threads_mod._THREAD_POOL = original_pool + + +def test_shutdown_thread_pool_is_idempotent() -> None: + threads_mod.shutdown_thread_pool(wait=True) + threads_mod.shutdown_thread_pool(wait=False) + + +def test_max_threads_zero_when_threading_unsupported(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(threads_mod, "_cpu_total", lambda: 4) + assert threads_mod._max_threads() == 0 + + +def test_determine_worker_count_requires_supported_backend(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(threads_mod, "_max_threads", lambda: 0) + with pytest.raises(RuntimeError, match="at least 8 CPU cores"): + threads_mod._determine_worker_count(None) + + +def test_configure_thread_pool_shuts_down_existing_pool() -> None: + original_pool = threads_mod._THREAD_POOL + original_workers = threads_mod._THREAD_POOL_WORKERS + try: + threads_mod.shutdown_thread_pool(wait=True) + threads_mod.configure_thread_pool(max_workers=2, preload=True) + first_pool = threads_mod._THREAD_POOL + assert first_pool is not None + + threads_mod.configure_thread_pool(max_workers=3, preload=True) + assert threads_mod._THREAD_POOL is not first_pool + finally: + threads_mod.shutdown_thread_pool(wait=True) + threads_mod._THREAD_POOL = original_pool + threads_mod._THREAD_POOL_WORKERS = original_workers + + +def test_ensure_thread_pool_resizes_existing_pool() -> None: + original_pool = threads_mod._THREAD_POOL + original_workers = threads_mod._THREAD_POOL_WORKERS + try: + threads_mod.shutdown_thread_pool(wait=True) + first_pool = threads_mod.ensure_thread_pool(max_workers=2) + assert first_pool is not None + + second_pool = threads_mod.ensure_thread_pool(max_workers=3) + assert second_pool is not first_pool + finally: + threads_mod.shutdown_thread_pool(wait=True) + threads_mod._THREAD_POOL = original_pool + threads_mod._THREAD_POOL_WORKERS = original_workers