diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h index b81e4211b08ff3..fd2e0a09584c1d 100644 --- a/Include/internal/pycore_lazyimportobject.h +++ b/Include/internal/pycore_lazyimportobject.h @@ -19,6 +19,10 @@ typedef struct { PyObject *lz_builtins; PyObject *lz_from; PyObject *lz_attr; + // Protected by the lazy object lock. + PyObject *lz_globals; + PyObject *lz_key; + PyObject *lz_resolved; // Frame information for the original import location. PyCodeObject *lz_code; // Code object where the lazy import was created. int lz_instr_offset; // Instruction offset where the lazy import was created. @@ -26,6 +30,14 @@ typedef struct { PyAPI_FUNC(PyObject *) _PyLazyImport_GetName(PyObject *lazy_import); +extern PyObject * _PyLazyImport_GetResolved(PyObject *lazy_import); +extern int _PyLazyImport_FinishResolve( + PyObject *lazy_import, PyObject *resolved); +PyAPI_FUNC(int) _PyLazyImport_ReplaceDictItemIfCurrent( + PyObject *lazy_import, PyObject *dict, PyObject *name, + PyObject *resolved); +PyAPI_FUNC(int) _PyLazyImport_SetGlobalBindingAndDictItem( + PyObject *lazy_import, PyObject *globals, PyObject *name); PyAPI_FUNC(PyObject *) _PyLazyImport_New( struct _PyInterpreterFrame *frame, PyObject *import_func, PyObject *from, PyObject *attr); diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index cf1d3eb793a8b9..57776798b9dc91 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -905,42 +905,380 @@ def test_direct_access_triggers_reification(self): result = test.test_lazy_import.data.globals_access.get_direct() self.assertIn("test.test_lazy_import.data.basic2", sys.modules) - def test_resolve_method_forces_reification(self): - """Calling resolve() on lazy proxy should force reification. + def test_resolve_method_preserves_binding_lifecycle(self): + code = textwrap.dedent(""" + import builtins + import threading + import types - Note: Must access lazy proxy from within a function to avoid automatic - reification by LOAD_NAME at module level. - """ + real_import = builtins.__import__ + calls = [] + + lazy import target_module as target + lazy import rebound_module as rebound + lazy import deleted_module as deleted + lazy import broken_module as broken + lazy from from_module import VALUE as value + lazy import threaded_module as threaded + + def custom_import(name, globals=None, locals=None, fromlist=None, level=0): + calls.append(name) + if name == "broken_module": + raise ImportError("boom") + module = types.ModuleType(name) + module.VALUE = name + return module + + builtins.__import__ = custom_import + try: + def check(): + g = globals() + + target_proxy = g["target"] + g["alias"] = target_proxy + resolved = target_proxy.resolve() + assert g["target"] is resolved + assert g["alias"] is target_proxy + assert target_proxy.resolve() is resolved + assert target is resolved + assert calls.count("target_module") == 1 + + sentinel = object() + rebound_proxy = g["rebound"] + g["rebound"] = sentinel + rebound_resolved = rebound_proxy.resolve() + assert g["rebound"] is sentinel + assert rebound_proxy.resolve() is rebound_resolved + assert calls.count("rebound_module") == 1 + + deleted_proxy = g["deleted"] + del g["deleted"] + deleted_proxy.resolve() + assert "deleted" not in g + assert calls.count("deleted_module") == 1 + + broken_proxy = g["broken"] + for _ in range(2): + try: + broken_proxy.resolve() + except ImportError as exc: + assert str(exc) == "boom" + else: + raise AssertionError("resolve() unexpectedly succeeded") + assert g["broken"] is broken_proxy + assert calls.count("broken_module") == 2 + + value_proxy = g["value"] + assert value_proxy.resolve() == "from_module" + assert g["value"] == "from_module" + assert calls.count("from_module") == 1 + + threaded_proxy = g["threaded"] + barrier = threading.Barrier(3) + results = [] + errors = [] + + def resolve_threaded(): + barrier.wait() + try: + results.append(threaded_proxy.resolve()) + except BaseException as exc: + errors.append(exc) + + threads = [ + threading.Thread(target=resolve_threaded) + for _ in range(2) + ] + for thread in threads: + thread.start() + barrier.wait() + for thread in threads: + thread.join() + + assert not errors, errors + assert len(results) == 2 + assert results[0] is results[1] + assert g["threaded"] is results[0] + assert calls.count("threaded_module") == 1 + + check() + finally: + builtins.__import__ = real_import + """) + assert_python_ok("-c", code) + + def test_resolve_writeback_reentrancy(self): code = textwrap.dedent(""" - import sys + import builtins import types - lazy from test.test_lazy_import.data.basic2 import x + real_import = builtins.__import__ + calls = [] + holder = [] + nested = [] + armed = False + raising = False + + class CollidingKey: + def __hash__(self): + return hash("target") + + def __eq__(self, other): + global armed + if armed: + armed = False + try: + holder[0].resolve() + except ImportCycleError: + nested.append("cycle") + else: + nested.append("resolved") + return False + + class RaisingKey: + def __hash__(self): + return hash("retry") + + def __eq__(self, other): + global raising + if raising: + raising = False + raise RuntimeError("writeback failed") + return False + + def custom_import(name, globals=None, locals=None, + fromlist=None, level=0): + if name in {"target_module", "retry_module"}: + module = types.ModuleType(name) + calls.append(module) + return module + return real_import(name, globals, locals, fromlist, level) + + globals()[CollidingKey()] = None + globals()[RaisingKey()] = None + lazy import target_module as target + lazy import retry_module as retry + holder.append(globals()["target"]) + holder.append(globals()["retry"]) + + builtins.__import__ = custom_import + try: + armed = True + resolved = holder[0].resolve() - assert 'test.test_lazy_import.data.basic2' not in sys.modules + assert nested == ["cycle"] + assert calls == [resolved] + assert holder[0].resolve() is resolved + assert globals()["target"] is resolved - def test_resolve(): - g = globals() - lazy_obj = g['x'] - assert type(lazy_obj) is types.LazyImportType, f"Expected lazy proxy, got {type(lazy_obj)}" + raising = True + try: + holder[1].resolve() + except RuntimeError as exc: + assert str(exc) == "writeback failed" + else: + raise AssertionError("writeback unexpectedly succeeded") + + assert [module.__name__ for module in calls] == [ + "target_module", "retry_module"] + assert globals()["retry"] is holder[1] + + retry_resolved = holder[1].resolve() + assert [module.__name__ for module in calls] == [ + "target_module", "retry_module", "retry_module"] + assert holder[1].resolve() is retry_resolved + assert globals()["retry"] is retry_resolved + finally: + builtins.__import__ = real_import + """) + assert_python_ok("-c", code) - resolved = lazy_obj.resolve() + def test_dict_subclass_globals_store_and_load(self): + code = textwrap.dedent(""" + import builtins + import types - # Now module should be loaded - assert 'test.test_lazy_import.data.basic2' in sys.modules - assert resolved == 42 # x is 42 in basic2.py - return True + class Globals(dict): + pass - assert test_resolve() - print("OK") + lazy import store_module as store_source + lazy import rebound_module as rebound_source + lazy import deleted_module as deleted_source + lazy import builtin_module as builtin_source + lazy import load_name_rebound_module as load_name_rebound_source + lazy import load_name_deleted_module as load_name_deleted_source + + sentinel = object() + calls = [] + + def store_template(value): + global target + target = value + + def read_rebound(): + return rebound_target + + def read_deleted(): + return deleted_target + + def read_builtin(): + return builtin_target + + def custom_import(name, globals=None, locals=None, fromlist=(), level=0): + calls.append(name) + if name == "rebound_module": + globals["rebound_target"] = sentinel + elif name == "deleted_module": + del globals["deleted_target"] + elif name == "load_name_rebound_module": + globals["target"] = sentinel + elif name == "load_name_deleted_module": + del globals["target"] + module = types.ModuleType(name) + module.VALUE = name + return module + + def check(): + module_globals = globals() + store_proxy = module_globals["store_source"] + rebound_proxy = module_globals["rebound_source"] + deleted_proxy = module_globals["deleted_source"] + builtin_proxy = module_globals["builtin_source"] + load_name_rebound_proxy = module_globals["load_name_rebound_source"] + load_name_deleted_proxy = module_globals["load_name_deleted_source"] + + store_globals = Globals() + store = types.FunctionType(store_template.__code__, store_globals) + store(store_proxy) + assert store_globals["target"] is store_proxy + + custom_builtins = dict(vars(builtins)) + custom_builtins["__import__"] = custom_import + custom_builtins["builtin_target"] = builtin_proxy + function_globals = Globals( + __builtins__=custom_builtins, + rebound_target=rebound_proxy, + deleted_target=deleted_proxy, + ) + + rebound_reader = types.FunctionType( + read_rebound.__code__, function_globals) + assert rebound_reader().VALUE == "rebound_module" + assert function_globals["rebound_target"] is sentinel + + deleted_reader = types.FunctionType( + read_deleted.__code__, function_globals) + assert deleted_reader().VALUE == "deleted_module" + assert "deleted_target" not in function_globals + + builtin_reader = types.FunctionType( + read_builtin.__code__, function_globals) + builtin_result = builtin_reader() + assert builtin_result.VALUE == "builtin_module" + assert builtin_reader() is builtin_result + assert "builtin_target" not in function_globals + + load_name_rebound_globals = Globals( + __builtins__=custom_builtins, + target=load_name_rebound_proxy, + ) + exec("result = target", load_name_rebound_globals, + load_name_rebound_globals) + assert load_name_rebound_globals["result"].VALUE == ( + "load_name_rebound_module") + assert load_name_rebound_globals["target"] is sentinel + + load_name_deleted_globals = Globals( + __builtins__=custom_builtins, + target=load_name_deleted_proxy, + ) + exec("result = target", load_name_deleted_globals, + load_name_deleted_globals) + assert load_name_deleted_globals["result"].VALUE == ( + "load_name_deleted_module") + assert "target" not in load_name_deleted_globals + + assert calls == [ + "rebound_module", + "deleted_module", + "builtin_module", + "load_name_rebound_module", + "load_name_deleted_module", + ] + + real_import = builtins.__import__ + builtins.__import__ = custom_import + try: + check() + finally: + builtins.__import__ = real_import """) - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True - ) - self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") - self.assertIn("OK", result.stdout) + assert_python_ok("-c", code) + + def test_lazy_import_rejects_dict_subclass_globals(self): + class Globals(dict): + pass + + namespace = Globals() + with self.assertRaisesRegex(TypeError, "requires exact dict globals"): + exec("lazy import json as value", namespace, namespace) + self.assertNotIn("value", namespace) + + sys.set_lazy_imports_filter(lambda *args: False) + exec("lazy import json as value", namespace, namespace) + self.assertIs(namespace["value"], sys.modules["json"]) + + def test_reification_preserves_changed_bindings(self): + code = textwrap.dedent(""" + import builtins + import types + + real_import = builtins.__import__ + calls = [] + sentinel = object() + + lazy import load_global_module as load_global_value + lazy import load_name_module as load_name_value + + holder = types.ModuleType("holder") + holder.__dict__["__builtins__"] = builtins + exec("lazy import module_attr_module as value", holder.__dict__) + + def custom_import(name, globals=None, locals=None, fromlist=None, level=0): + calls.append(name) + if name == "load_global_module": + globals["load_global_value"] = sentinel + elif name == "load_name_module": + del globals["load_name_value"] + elif name == "module_attr_module": + holder.__dict__["value"] = sentinel + module = types.ModuleType(name) + module.VALUE = name + return module + + builtins.__import__ = custom_import + try: + def load_global(): + return load_global_value + + assert load_global().VALUE == "load_global_module" + assert globals()["load_global_value"] is sentinel + + load_name_result = load_name_value + assert load_name_result.VALUE == "load_name_module" + assert "load_name_value" not in globals() + + assert holder.value.VALUE == "module_attr_module" + assert holder.__dict__["value"] is sentinel + assert calls == [ + "load_global_module", + "load_name_module", + "module_attr_module", + ] + finally: + builtins.__import__ = real_import + """) + assert_python_ok("-c", code) def test_add_lazy_to_globals(self): code = textwrap.dedent(""" diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-29-03-18-22.gh-issue-152298.X3zQpA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-29-03-18-22.gh-issue-152298.X3zQpA.rst new file mode 100644 index 00000000000000..c934afa151ae38 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-29-03-18-22.gh-issue-152298.X3zQpA.rst @@ -0,0 +1,3 @@ +Fix :meth:`!types.LazyImportType.resolve` so resolving a module-level lazy +import replaces the original lazy proxy in the module globals. Lazy import +creation now requires an exact dictionary for the globals namespace. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 708aa4d966e535..cfb0902e3776ce 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -10336,10 +10336,14 @@ _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - int err = PyDict_SetItem(GLOBALS(), name, l_v); - _PyFrame_StackPointerInvalidate(frame); + int err = 0; + if (PyDict_CheckExact(GLOBALS())) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + err = _PyLazyImport_ReplaceDictItemIfCurrent( + v_o, GLOBALS(), name, l_v); + _PyFrame_StackPointerInvalidate(frame); + } if (err < 0) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); @@ -12525,10 +12529,22 @@ } } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - _PyFrame_StackPointerInvalidate(frame); + PyObject *value = PyStackRef_AsPyObjectBorrow(v); + if (PyLazyImport_CheckExact(value) && + PyDict_CheckExact(GLOBALS())) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = _PyLazyImport_SetGlobalBindingAndDictItem( + value, GLOBALS(), name); + _PyFrame_StackPointerInvalidate(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = PyDict_SetItem(GLOBALS(), name, value); + _PyFrame_StackPointerInvalidate(frame); + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index fa1eb25047d961..6c7132f432b72b 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -2,6 +2,8 @@ #include "Python.h" #include "pycore_ceval.h" +#include "pycore_critical_section.h" +#include "pycore_dict.h" #include "pycore_frame.h" #include "pycore_import.h" #include "pycore_interpframe.h" @@ -33,6 +35,9 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, PyObject *name m->lz_builtins = Py_XNewRef(builtins); m->lz_from = Py_NewRef(name); m->lz_attr = Py_XNewRef(fromlist); + m->lz_globals = NULL; + m->lz_key = NULL; + m->lz_resolved = NULL; // Capture frame information for the original import location. m->lz_code = NULL; @@ -58,6 +63,9 @@ lazy_import_traverse(PyObject *op, visitproc visit, void *arg) Py_VISIT(m->lz_builtins); Py_VISIT(m->lz_from); Py_VISIT(m->lz_attr); + Py_VISIT(m->lz_globals); + Py_VISIT(m->lz_key); + Py_VISIT(m->lz_resolved); Py_VISIT(m->lz_code); return 0; } @@ -69,6 +77,9 @@ lazy_import_clear(PyObject *op) Py_CLEAR(m->lz_builtins); Py_CLEAR(m->lz_from); Py_CLEAR(m->lz_attr); + Py_CLEAR(m->lz_globals); + Py_CLEAR(m->lz_key); + Py_CLEAR(m->lz_resolved); Py_CLEAR(m->lz_code); return 0; } @@ -116,6 +127,165 @@ _PyLazyImport_GetName(PyObject *op) return lazy_import_name(lazy_import); } +PyObject * +_PyLazyImport_GetResolved(PyObject *op) +{ + PyLazyImportObject *m = PyLazyImportObject_CAST(op); + assert(PyLazyImport_CheckExact(op)); + + PyObject *resolved = NULL; + Py_BEGIN_CRITICAL_SECTION(op); + resolved = Py_XNewRef(m->lz_resolved); + Py_END_CRITICAL_SECTION(); + return resolved; +} + +int +_PyLazyImport_SetGlobalBindingAndDictItem(PyObject *op, PyObject *globals, + PyObject *name) +{ + assert(PyLazyImport_CheckExact(op)); + assert(PyDict_CheckExact(globals)); + + PyLazyImportObject *m = PyLazyImportObject_CAST(op); + Py_hash_t hash = PyObject_Hash(name); + if (hash == -1) { + return -1; + } + + PyObject *discard_globals = NULL; + PyObject *discard_key = NULL; + int err; + int recorded = 0; + + Py_BEGIN_CRITICAL_SECTION2(op, globals); + if (m->lz_key == NULL && m->lz_resolved == NULL) { + // Record the owner binding before publishing the proxy. resolve() + // may update only this key; aliases must not retarget it. + m->lz_globals = Py_NewRef(globals); + m->lz_key = Py_NewRef(name); + recorded = 1; + } + err = _PyDict_SetItem_KnownHash_LockHeld( + (PyDictObject *)globals, name, op, hash); + if (err < 0 && recorded) { + discard_globals = m->lz_globals; + discard_key = m->lz_key; + m->lz_globals = NULL; + m->lz_key = NULL; + } + Py_END_CRITICAL_SECTION2(); + + Py_XDECREF(discard_globals); + Py_XDECREF(discard_key); + return err; +} + +static int +lazy_import_replace_dict_item_if_current(PyObject *op, PyObject *globals, + PyObject *key, Py_hash_t key_hash, + PyObject *resolved) +{ + assert(PyLazyImport_CheckExact(op)); + assert(PyDict_CheckExact(globals)); + assert(!PyLazyImport_CheckExact(resolved)); + + PyObject *current = NULL; + int err = 0; + + Py_BEGIN_CRITICAL_SECTION(globals); + int found = _PyDict_GetItemRef_KnownHash_LockHeld( + (PyDictObject *)globals, key, key_hash, ¤t); + if (found < 0) { + err = -1; + } + else if (found && current == op) { + err = _PyDict_SetItem_KnownHash_LockHeld( + (PyDictObject *)globals, key, resolved, key_hash); + } + Py_END_CRITICAL_SECTION(); + + Py_XDECREF(current); + return err; +} + +int +_PyLazyImport_ReplaceDictItemIfCurrent(PyObject *op, PyObject *globals, + PyObject *key, PyObject *resolved) +{ + assert(PyLazyImport_CheckExact(op)); + assert(PyDict_CheckExact(globals)); + + Py_hash_t key_hash = PyObject_Hash(key); + if (key_hash == -1) { + return -1; + } + return lazy_import_replace_dict_item_if_current( + op, globals, key, key_hash, resolved); +} + +int +_PyLazyImport_FinishResolve(PyObject *op, PyObject *resolved) +{ + PyLazyImportObject *m = PyLazyImportObject_CAST(op); + PyObject *globals = NULL; + PyObject *key = NULL; + int already_resolved = 0; + + assert(PyLazyImport_CheckExact(op)); + assert(!PyLazyImport_CheckExact(resolved)); + + Py_BEGIN_CRITICAL_SECTION(op); + if (m->lz_resolved != NULL) { + already_resolved = 1; + } + else if (m->lz_globals != NULL && m->lz_key != NULL) { + globals = Py_NewRef(m->lz_globals); + key = Py_NewRef(m->lz_key); + } + Py_END_CRITICAL_SECTION(); + + if (already_resolved) { + return 0; + } + + if (globals != NULL) { + assert(key != NULL); + assert(PyDict_CheckExact(globals)); + + int err = _PyLazyImport_ReplaceDictItemIfCurrent( + op, globals, key, resolved); + if (err < 0) { + Py_DECREF(globals); + Py_DECREF(key); + return err; + } + } + + PyObject *discard_globals = NULL; + PyObject *discard_key = NULL; + Py_BEGIN_CRITICAL_SECTION(op); + if (m->lz_resolved == NULL) { + m->lz_resolved = Py_NewRef(resolved); + } + if (globals != NULL && + m->lz_globals == globals && + m->lz_key == key) + { + discard_globals = m->lz_globals; + discard_key = m->lz_key; + m->lz_globals = NULL; + m->lz_key = NULL; + } + Py_END_CRITICAL_SECTION(); + + Py_XDECREF(discard_globals); + Py_XDECREF(discard_key); + Py_XDECREF(globals); + Py_XDECREF(key); + return 0; +} + static PyObject * lazy_import_resolve(PyObject *self, PyObject *args) { @@ -137,9 +307,10 @@ PyDoc_STRVAR(lazy_import_doc, "\n" "Represents a lazy import that will be resolved on first use.\n" "\n" -"Instances of this object accessed from the global scope will be\n" -"automatically imported based upon their name and then replaced with\n" -"the imported value."); +"A successful resolution is cached. Instances of this object accessed\n" +"from the global scope will be automatically imported based upon their\n" +"name. The original global is replaced only if it still refers to this\n" +"lazy import object."); PyTypeObject PyLazyImport_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index f447403ef31b43..aa7234f856ad52 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1368,7 +1368,9 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) return NULL; } - if (PyDict_SetItem(m->md_dict, name, new_value) < 0) { + if (_PyLazyImport_ReplaceDictItemIfCurrent( + attr, m->md_dict, name, new_value) < 0) + { Py_CLEAR(new_value); } Py_DECREF(attr); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index fa64255be00e75..a207613ef65fe8 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -299,15 +299,16 @@ __lazy_import__ as builtin___lazy_import__ Lazily imports a module. -Returns either the module to be imported or a imp.lazy_module object -which indicates the module to be lazily imported. +Returns either the imported object or a proxy for the lazy import. + +Creating a lazy import proxy requires globals to be an exact dict. [clinic start generated code]*/ static PyObject * builtin___lazy_import___impl(PyObject *module, PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) -/*[clinic end generated code: output=300f1771094b9e8c input=9c85cccd6a885b9b]*/ +/*[clinic end generated code: output=300f1771094b9e8c input=acafb35a7969f5ea]*/ { PyObject *builtins; PyThreadState *tstate = PyThreadState_GET(); @@ -324,6 +325,8 @@ builtin___lazy_import___impl(PyObject *module, PyObject *name, locals = globals; } + // Dict subclasses remain valid for the filter-controlled eager fallback; + // the proxy creation path enforces its stronger binding-owner contract. if (!PyDict_Check(globals)) { PyErr_Format(PyExc_TypeError, "expect dict for globals, got %T", globals); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6d5ae1b074db5b..11a0f23ce69b2d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2174,7 +2174,16 @@ dummy_func( } } else { - err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + PyObject *value = PyStackRef_AsPyObjectBorrow(v); + if (PyLazyImport_CheckExact(value) && + PyDict_CheckExact(GLOBALS())) + { + err = _PyLazyImport_SetGlobalBindingAndDictItem( + value, GLOBALS(), name); + } + else { + err = PyDict_SetItem(GLOBALS(), name, value); + } PyStackRef_CLOSE(v); } ERROR_IF(err < 0); @@ -2257,7 +2266,11 @@ dummy_func( Py_DECREF(v_o); ERROR_IF(true); } - int err = PyDict_SetItem(GLOBALS(), name, l_v); + int err = 0; + if (PyDict_CheckExact(GLOBALS())) { + err = _PyLazyImport_ReplaceDictItemIfCurrent( + v_o, GLOBALS(), name, l_v); + } if (err < 0) { Py_DECREF(v_o); Py_DECREF(l_v); diff --git a/Python/ceval.c b/Python/ceval.c index f3f03b28112137..1443baccadffa3 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1,6 +1,7 @@ /* Execute compiled code */ #include "ceval.h" +#include "pycore_lazyimportobject.h" #include "pycore_long.h" int @@ -1171,6 +1172,11 @@ int _PyEval_StoreName(PyThreadState *tstate, _PyStackRef v, PyObject *name, PyOb PyObject *v_o = PyStackRef_AsPyObjectBorrow(v); if (PyDict_CheckExact(ns)) { + if (ns == PyEval_GetGlobals() && + PyLazyImport_CheckExact(v_o)) + { + return _PyLazyImport_SetGlobalBindingAndDictItem(v_o, ns, name); + } return PyDict_SetItem(ns, name, v_o); } @@ -3682,13 +3688,18 @@ _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name PyObject *res_o = PyStackRef_AsPyObjectBorrow(*writeto); if (res_o != NULL && PyLazyImport_CheckExact(res_o)) { PyObject *l_v = _PyImport_LoadLazyImportTstate(PyThreadState_GET(), res_o); - PyStackRef_CLOSE(writeto[0]); if (l_v == NULL) { assert(PyErr_Occurred()); + PyStackRef_CLOSE(writeto[0]); *writeto = PyStackRef_NULL; return; } - int err = PyDict_SetItem(globals, name, l_v); + int err = 0; + if (PyDict_CheckExact(globals)) { + err = _PyLazyImport_ReplaceDictItemIfCurrent( + res_o, globals, name, l_v); + } + PyStackRef_CLOSE(writeto[0]); if (err < 0) { Py_DECREF(l_v); *writeto = PyStackRef_NULL; diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 4a38e0df61708c..f110ee0af5fcc9 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -121,8 +121,9 @@ PyDoc_STRVAR(builtin___lazy_import____doc__, "\n" "Lazily imports a module.\n" "\n" -"Returns either the module to be imported or a imp.lazy_module object\n" -"which indicates the module to be lazily imported."); +"Returns either the imported object or a proxy for the lazy import.\n" +"\n" +"Creating a lazy import proxy requires globals to be an exact dict."); #define BUILTIN___LAZY_IMPORT___METHODDEF \ {"__lazy_import__", _PyCFunction_CAST(builtin___lazy_import__), METH_FASTCALL|METH_KEYWORDS, builtin___lazy_import____doc__}, @@ -1387,4 +1388,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=84efa9c5cc737ce5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0e55fd33f99065ac input=a9049054013a1b77]*/ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6be53213966678..e5b77cd5eba43e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -10033,13 +10033,28 @@ } } else { - stack_pointer[0] = v; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - _PyFrame_StackPointerInvalidate(frame); + PyObject *value = PyStackRef_AsPyObjectBorrow(v); + if (PyLazyImport_CheckExact(value) && + PyDict_CheckExact(GLOBALS())) + { + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = _PyLazyImport_SetGlobalBindingAndDictItem( + value, GLOBALS(), name); + _PyFrame_StackPointerInvalidate(frame); + } + else { + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = PyDict_SetItem(GLOBALS(), name, value); + _PyFrame_StackPointerInvalidate(frame); + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -10164,10 +10179,14 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - int err = PyDict_SetItem(GLOBALS(), name, l_v); - _PyFrame_StackPointerInvalidate(frame); + int err = 0; + if (PyDict_CheckExact(GLOBALS())) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + err = _PyLazyImport_ReplaceDictItemIfCurrent( + v_o, GLOBALS(), name, l_v); + _PyFrame_StackPointerInvalidate(frame); + } if (err < 0) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d336b9fbcd5712..ac39a4f0c0ea53 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -10334,10 +10334,14 @@ _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - int err = PyDict_SetItem(GLOBALS(), name, l_v); - _PyFrame_StackPointerInvalidate(frame); + int err = 0; + if (PyDict_CheckExact(GLOBALS())) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + err = _PyLazyImport_ReplaceDictItemIfCurrent( + v_o, GLOBALS(), name, l_v); + _PyFrame_StackPointerInvalidate(frame); + } if (err < 0) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); @@ -12522,10 +12526,22 @@ } } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - _PyFrame_StackPointerInvalidate(frame); + PyObject *value = PyStackRef_AsPyObjectBorrow(v); + if (PyLazyImport_CheckExact(value) && + PyDict_CheckExact(GLOBALS())) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = _PyLazyImport_SetGlobalBindingAndDictItem( + value, GLOBALS(), name); + _PyFrame_StackPointerInvalidate(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = PyDict_SetItem(GLOBALS(), name, value); + _PyFrame_StackPointerInvalidate(frame); + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/import.c b/Python/import.c index 63e23e21beb126..be5b85d676441d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3897,6 +3897,12 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) // Acquire the global import lock to serialize reification _PyImport_AcquireLock(interp); + obj = _PyLazyImport_GetResolved(lazy_import); + if (obj != NULL) { + _PyImport_ReleaseLock(interp); + return obj; + } + // Check if we are already importing this module, if so, then we want to // return an error that indicates we've hit a cycle which will indicate // the value isn't yet available. @@ -4090,6 +4096,14 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) } ok: + if (obj != NULL) { + // Keep the proxy marked as importing while publishing the result. + // Updating the globals dict can invoke a colliding key's equality + // method, which may otherwise re-enter resolution through this proxy. + if (_PyLazyImport_FinishResolve(lazy_import, obj) < 0) { + Py_CLEAR(obj); + } + } if (PySet_Discard(importing, lazy_import) < 0) { Py_CLEAR(obj); } @@ -4567,6 +4581,15 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, } } + // Reification conditionally replaces the original binding by identity. + // Mapping overrides cannot provide that atomic check-and-replace contract. + if (!PyDict_CheckExact(globals)) { + Py_DECREF(abs_name); + PyErr_SetString(PyExc_TypeError, + "'lazy import' requires exact dict globals"); + return NULL; + } + // here, 'filter' is either NULL or is equivalent to a borrowed reference if (fromlist && PyUnicode_Check(fromlist)) { fromlist = PyTuple_Pack(1, fromlist);