|
| 1 | +"""Reproduction and regression tests for the Windows atomic-write crash (#11). |
| 2 | +
|
| 3 | +`write_text_atomic` renames a temp file over the destination. On Windows that |
| 4 | +rename fails with ``PermissionError`` (``WinError 5`` access-denied, or ``32`` |
| 5 | +sharing-violation) whenever another process — antivirus, the search indexer, |
| 6 | +OneDrive — briefly holds the destination or the temp file open without |
| 7 | +``FILE_SHARE_DELETE``. Under ``-j 8`` the checkpoint is rewritten on every |
| 8 | +completed function, so such a collision eventually aborts the whole export. |
| 9 | +
|
| 10 | +The portable test reproduces the mechanism on any platform by monkeypatching the |
| 11 | +rename; the Windows test reproduces the exact production failure with a real lock. |
| 12 | +""" |
| 13 | + |
| 14 | +from __future__ import annotations |
| 15 | + |
| 16 | +import sys |
| 17 | +from pathlib import Path |
| 18 | + |
| 19 | +import pytest |
| 20 | + |
| 21 | +from tocode.metadata import write_text_atomic |
| 22 | + |
| 23 | + |
| 24 | +def _win_permission_error(winerror: int) -> PermissionError: |
| 25 | + exc = PermissionError(winerror, "Access is denied") |
| 26 | + # Mirror the OSError.winerror attribute CPython sets on Windows so the code |
| 27 | + # under test can distinguish transient lock errors from real ones. |
| 28 | + exc.winerror = winerror # type: ignore[attr-defined] |
| 29 | + return exc |
| 30 | + |
| 31 | + |
| 32 | +def _tmp_siblings(path: Path) -> list[Path]: |
| 33 | + return [p for p in path.parent.iterdir() if p.name != path.name] |
| 34 | + |
| 35 | + |
| 36 | +def test_replace_retries_past_transient_lock(tmp_path, monkeypatch) -> None: |
| 37 | + """A few transient WinError 5 failures must not abort the write.""" |
| 38 | + target = tmp_path / "checkpoint.json" |
| 39 | + real_replace = Path.replace |
| 40 | + calls = {"n": 0} |
| 41 | + |
| 42 | + def flaky_replace(self: Path, dst): |
| 43 | + calls["n"] += 1 |
| 44 | + if calls["n"] <= 3: # transient lock clears after a few attempts |
| 45 | + raise _win_permission_error(5) |
| 46 | + return real_replace(self, dst) |
| 47 | + |
| 48 | + monkeypatch.setattr(Path, "replace", flaky_replace) |
| 49 | + monkeypatch.setattr("tocode.metadata.time.sleep", lambda _s: None) |
| 50 | + |
| 51 | + write_text_atomic(target, "payload\n") |
| 52 | + |
| 53 | + assert calls["n"] == 4 # 3 failures + 1 success |
| 54 | + assert target.read_text(encoding="utf-8") == "payload\n" |
| 55 | + # No orphan .tmp left behind. |
| 56 | + assert _tmp_siblings(target) == [] |
| 57 | + |
| 58 | + |
| 59 | +def test_replace_gives_up_and_cleans_tmp_when_lock_never_clears( |
| 60 | + tmp_path, monkeypatch |
| 61 | +) -> None: |
| 62 | + """A permanent lock still raises, but must not litter .tmp orphans.""" |
| 63 | + target = tmp_path / "checkpoint.json" |
| 64 | + |
| 65 | + def always_locked(self: Path, dst): |
| 66 | + raise _win_permission_error(5) |
| 67 | + |
| 68 | + monkeypatch.setattr(Path, "replace", always_locked) |
| 69 | + monkeypatch.setattr("tocode.metadata.time.sleep", lambda _s: None) |
| 70 | + |
| 71 | + with pytest.raises(PermissionError): |
| 72 | + write_text_atomic(target, "payload\n") |
| 73 | + |
| 74 | + assert not target.exists() |
| 75 | + assert _tmp_siblings(target) == [] |
| 76 | + |
| 77 | + |
| 78 | +def test_non_windows_permission_error_is_not_retried(tmp_path, monkeypatch) -> None: |
| 79 | + """A PermissionError without a transient winerror re-raises immediately.""" |
| 80 | + target = tmp_path / "checkpoint.json" |
| 81 | + calls = {"n": 0} |
| 82 | + |
| 83 | + def denied(self: Path, dst): |
| 84 | + calls["n"] += 1 |
| 85 | + raise PermissionError(13, "Permission denied") # e.g. read-only dir |
| 86 | + |
| 87 | + monkeypatch.setattr(Path, "replace", denied) |
| 88 | + monkeypatch.setattr("tocode.metadata.time.sleep", lambda _s: None) |
| 89 | + |
| 90 | + with pytest.raises(PermissionError): |
| 91 | + write_text_atomic(target, "payload\n") |
| 92 | + |
| 93 | + assert calls["n"] == 1 # not retried |
| 94 | + |
| 95 | + |
| 96 | +# --- Faithful Windows reproduction with a real file lock ------------------ |
| 97 | + |
| 98 | +_WINDOWS_ONLY = pytest.mark.skipif( |
| 99 | + sys.platform != "win32", reason="reproduces a Windows file-sharing error" |
| 100 | +) |
| 101 | + |
| 102 | + |
| 103 | +def _open_read_share_no_delete(path: Path): |
| 104 | + """Open ``path`` the way a scanner does: read-share, no delete-share. |
| 105 | +
|
| 106 | + Returns a Windows HANDLE that blocks anyone from replacing/deleting the file |
| 107 | + until it is closed. Mirrors antivirus / OneDrive real-time access. |
| 108 | + """ |
| 109 | + import ctypes |
| 110 | + |
| 111 | + GENERIC_READ = 0x80000000 |
| 112 | + FILE_SHARE_READ = 0x00000001 |
| 113 | + OPEN_EXISTING = 3 |
| 114 | + INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value |
| 115 | + |
| 116 | + # windll is Windows-only (absent from the ctypes stubs on other platforms), |
| 117 | + # so reach it dynamically to keep static analysis happy on the Linux CI. A |
| 118 | + # HANDLE is a void*, so use c_void_p rather than the win-only wintypes. |
| 119 | + kernel32 = getattr(ctypes, "windll").kernel32 |
| 120 | + kernel32.CreateFileW.restype = ctypes.c_void_p |
| 121 | + handle = kernel32.CreateFileW( |
| 122 | + str(path), |
| 123 | + GENERIC_READ, |
| 124 | + FILE_SHARE_READ, # deliberately no FILE_SHARE_DELETE |
| 125 | + None, |
| 126 | + OPEN_EXISTING, |
| 127 | + 0, |
| 128 | + None, |
| 129 | + ) |
| 130 | + if handle == INVALID_HANDLE_VALUE: |
| 131 | + raise OSError(f"CreateFileW failed to lock {path}") |
| 132 | + return handle |
| 133 | + |
| 134 | + |
| 135 | +def _close_handle(handle) -> None: |
| 136 | + import ctypes |
| 137 | + |
| 138 | + getattr(ctypes, "windll").kernel32.CloseHandle(handle) |
| 139 | + |
| 140 | + |
| 141 | +@_WINDOWS_ONLY |
| 142 | +def test_windows_permanent_lock_reproduces_winerror5(tmp_path) -> None: |
| 143 | + target = tmp_path / "checkpoint.json" |
| 144 | + target.write_text("old\n", encoding="utf-8") |
| 145 | + handle = _open_read_share_no_delete(target) |
| 146 | + try: |
| 147 | + with pytest.raises(PermissionError) as info: |
| 148 | + write_text_atomic(target, "new\n") |
| 149 | + assert getattr(info.value, "winerror", None) in (5, 32) |
| 150 | + finally: |
| 151 | + _close_handle(handle) |
| 152 | + |
| 153 | + |
| 154 | +@_WINDOWS_ONLY |
| 155 | +def test_windows_transient_lock_recovers(tmp_path) -> None: |
| 156 | + import threading |
| 157 | + import time |
| 158 | + |
| 159 | + target = tmp_path / "checkpoint.json" |
| 160 | + target.write_text("old\n", encoding="utf-8") |
| 161 | + handle = _open_read_share_no_delete(target) |
| 162 | + |
| 163 | + # Release the lock shortly after the write starts, as a scanner would. |
| 164 | + threading.Timer(0.15, lambda: _close_handle(handle)).start() |
| 165 | + |
| 166 | + started = time.monotonic() |
| 167 | + write_text_atomic(target, "new\n") # retries until the lock clears |
| 168 | + assert target.read_text(encoding="utf-8") == "new\n" |
| 169 | + assert time.monotonic() - started >= 0.1 |
| 170 | + assert _tmp_siblings(target) == [] |
0 commit comments