From cdd7ca2c495754de273405911fef02809f23eb4c Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sun, 6 Sep 2026 12:38:40 +0530 Subject: [PATCH 1/3] Implement anext() in Python instead of C Remove the C implementation of the anext() builtin and the anext_awaitable helper type. Instead, compile a small Python source at interpreter startup and copy the resulting anext() function into the builtins dict. The Python version keeps the C semantics: __anext__ is looked up on the type and called eagerly, the one-argument form returns the awaitable unchanged, and the two-argument form wraps it in a coroutine that returns the default on StopAsyncIteration. The source is executed from pycore_init_builtins() rather than _PyBuiltin_Init(), since running bytecode requires the interpreter's common constants, which are only set up after the builtins module is created. --- .../pycore_global_objects_fini_generated.h | 2 + Include/internal/pycore_global_strings.h | 2 + Include/internal/pycore_pylifecycle.h | 1 + .../internal/pycore_runtime_init_generated.h | 2 + .../internal/pycore_unicodeobject_generated.h | 8 + Lib/test/test_asyncio/test_graph.py | 40 ++++ Lib/test/test_inspect/test_inspect.py | 2 +- Objects/iterobject.c | 182 ------------------ Objects/object.c | 2 - Python/bltinmodule.c | 152 ++++++++++----- Python/clinic/bltinmodule.c.h | 40 +--- Python/pylifecycle.c | 4 + Tools/c-analyzer/cpython/globals-to-fix.tsv | 2 - 13 files changed, 166 insertions(+), 273 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 4fd6c618fb74404..3cf4e49b3b0bfa6 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1310,6 +1310,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(bytes_characters)[253]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(bytes_characters)[254]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(bytes_characters)[255]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_builtins)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_dictcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_genexpr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_lambda)); @@ -1587,6 +1588,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alphabet)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(anext)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(any)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arg)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 5b35c53e0aa03b1..28350af3e80af8f 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -30,6 +30,7 @@ extern "C" { /* The following is auto-generated by Tools/build/generate_global_objects.py. */ struct _Py_global_strings { struct { + STRUCT_FOR_STR(anon_builtins, "") STRUCT_FOR_STR(anon_dictcomp, "") STRUCT_FOR_STR(anon_genexpr, "") STRUCT_FOR_STR(anon_lambda, "") @@ -310,6 +311,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(all_threads) STRUCT_FOR_ID(allow_code) STRUCT_FOR_ID(alphabet) + STRUCT_FOR_ID(anext) STRUCT_FOR_ID(any) STRUCT_FOR_ID(append) STRUCT_FOR_ID(arg) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index ab627c28c1fa5ee..bfc94e3e8529b75 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -26,6 +26,7 @@ extern int _Py_IsLocaleCoercionTarget(const char *ctype_loc); extern void _Py_InitVersion(void); extern PyStatus _PyFaulthandler_Init(int enable); extern PyObject * _PyBuiltin_Init(PyInterpreterState *interp); +extern int _PyBuiltin_InitPythonFunctions(PyObject *dict); extern PyStatus _PySys_Create( PyThreadState *tstate, PyObject **sysmod_p); diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index c80925f020186ba..43e6b974d419a29 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1305,6 +1305,7 @@ extern "C" { } #define _Py_str_literals_INIT { \ + INIT_STR(anon_builtins, ""), \ INIT_STR(anon_dictcomp, ""), \ INIT_STR(anon_genexpr, ""), \ INIT_STR(anon_lambda, ""), \ @@ -1585,6 +1586,7 @@ extern "C" { INIT_ID(all_threads), \ INIT_ID(allow_code), \ INIT_ID(alphabet), \ + INIT_ID(anext), \ INIT_ID(any), \ INIT_ID(append), \ INIT_ID(arg), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index b30cfc678de1cd7..d057702eef4cb06 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1020,6 +1020,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(anext); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(any); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3476,6 +3480,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_builtins); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_dictcomp); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index a442a346ff06d91..1326ef50c149b0b 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -148,6 +148,46 @@ async def main(): 'async generator CallStackTestBase.test_stack_async_gen..gen()', stack_for_gen_nested_call[1]) + async def test_stack_anext_default(self): + # anext() with a default wraps the awaitable in a coroutine, so the + # call graph of a suspended task sees through it into __anext__(). + + loop = asyncio.get_running_loop() + blocker = loop.create_future() + + async def inner(): + await blocker + + class AIter: + def __aiter__(self): + return self + + async def __anext__(self): + await inner() + return 1 + + async def main(): + await anext(AIter(), None) + + task = asyncio.create_task(main(), name='anext task') + await asyncio.sleep(0) + try: + stack = capture_test_stack(fut=task) + finally: + blocker.set_result(None) + await task + + self.assertEqual(stack[0], [ + 'T', + [ + 'a inner', + 'a __anext__', + 'a _anext_with_default', + 'a main', + ], + [] + ]) + def test_ag_frame_used_for_async_generator(self): # Regression test for gh-148736: the ag_await branch of # _build_graph_for_future must read ag_frame, not cr_frame. diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index df5843abfcb8753..25276fc40cb0287 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -6174,7 +6174,7 @@ def test_builtins_have_signatures(self): "next", "vars"} no_signature |= needs_groups # These have unrepresentable parameter default values of NULL - unsupported_signature = {"anext", "aiter", "iter"} + unsupported_signature = {"aiter", "iter"} # These need *args support in Argument Clinic needs_varargs = {"min", "max", "__build_class__"} no_signature |= needs_varargs diff --git a/Objects/iterobject.c b/Objects/iterobject.c index b5783c92c8eb689..2d5e3709a27dfb0 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -403,33 +403,6 @@ PyTypeObject PyCallIter_Type = { /* -------------------------------------- */ -typedef struct { - PyObject_HEAD - PyObject *wrapped; - PyObject *default_value; -} anextawaitableobject; - -#define anextawaitableobject_CAST(op) ((anextawaitableobject *)(op)) - -static void -anextawaitable_dealloc(PyObject *op) -{ - anextawaitableobject *obj = anextawaitableobject_CAST(op); - _PyObject_GC_UNTRACK(obj); - Py_XDECREF(obj->wrapped); - Py_XDECREF(obj->default_value); - PyObject_GC_Del(obj); -} - -static int -anextawaitable_traverse(PyObject *op, visitproc visit, void *arg) -{ - anextawaitableobject *obj = anextawaitableobject_CAST(op); - Py_VISIT(obj->wrapped); - Py_VISIT(obj->default_value); - return 0; -} - static PyObject * awaitable_getiter(PyObject *owner, PyObject *wrapped) { @@ -461,99 +434,6 @@ awaitable_getiter(PyObject *owner, PyObject *wrapped) return awaitable; } -static PyObject * -anextawaitable_iternext(PyObject *op) -{ - /* Consider the following class: - * - * class A: - * async def __anext__(self): - * ... - * a = A() - * - * Then `await anext(a)` should call - * a.__anext__().__await__().__next__() - * - * On the other hand, given - * - * async def agen(): - * yield 1 - * yield 2 - * gen = agen() - * - * Then `await anext(gen)` can just call - * gen.__anext__().__next__() - */ - anextawaitableobject *obj = anextawaitableobject_CAST(op); - PyObject *awaitable = awaitable_getiter(op, obj->wrapped); - if (awaitable == NULL) { - return NULL; - } - PyObject *result = (*Py_TYPE(awaitable)->tp_iternext)(awaitable); - Py_DECREF(awaitable); - if (result != NULL) { - return result; - } - if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) { - PyErr_Clear(); - _PyGen_SetStopIterationValue(obj->default_value); - } - return NULL; -} - - -static PyObject * -anextawaitable_proxy(anextawaitableobject *obj, char *meth, PyObject *arg) -{ - PyObject *awaitable = awaitable_getiter((PyObject *)obj, obj->wrapped); - if (awaitable == NULL) { - return NULL; - } - // When specified, 'arg' may be a tuple (if coming from a METH_VARARGS - // method) or a single object (if coming from a METH_O method). - PyObject *ret = arg == NULL - ? PyObject_CallMethod(awaitable, meth, NULL) - : PyObject_CallMethod(awaitable, meth, "O", arg); - Py_DECREF(awaitable); - if (ret != NULL) { - return ret; - } - if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) { - /* `anextawaitableobject` is only used by `anext()` when - * a default value is provided. So when we have a StopAsyncIteration - * exception we replace it with a `StopIteration(default)`, as if - * it was the return value of `__anext__()` coroutine. - */ - PyErr_Clear(); - _PyGen_SetStopIterationValue(obj->default_value); - } - return NULL; -} - - -static PyObject * -anextawaitable_send(PyObject *op, PyObject *arg) -{ - anextawaitableobject *obj = anextawaitableobject_CAST(op); - return anextawaitable_proxy(obj, "send", arg); -} - - -static PyObject * -anextawaitable_throw(PyObject *op, PyObject *args) -{ - anextawaitableobject *obj = anextawaitableobject_CAST(op); - return anextawaitable_proxy(obj, "throw", args); -} - - -static PyObject * -anextawaitable_close(PyObject *op, PyObject *Py_UNUSED(dummy)) -{ - anextawaitableobject *obj = anextawaitableobject_CAST(op); - return anextawaitable_proxy(obj, "close", NULL); -} - PyDoc_STRVAR(send_doc, "send(arg) -> send 'arg' into the wrapped iterator,\n\ @@ -574,68 +454,6 @@ PyDoc_STRVAR(close_doc, "close() -> raise GeneratorExit inside generator."); -static PyMethodDef anextawaitable_methods[] = { - {"send", anextawaitable_send, METH_O, send_doc}, - {"throw", anextawaitable_throw, METH_VARARGS, throw_doc}, - {"close", anextawaitable_close, METH_NOARGS, close_doc}, - {NULL, NULL} /* Sentinel */ -}; - - -static PyAsyncMethods anextawaitable_as_async = { - PyObject_SelfIter, /* am_await */ - 0, /* am_aiter */ - 0, /* am_anext */ - 0, /* am_send */ -}; - -PyTypeObject _PyAnextAwaitable_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "anext_awaitable", /* tp_name */ - sizeof(anextawaitableobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - anextawaitable_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - &anextawaitable_as_async, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - 0, /* tp_doc */ - anextawaitable_traverse, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ - anextawaitable_iternext, /* tp_iternext */ - anextawaitable_methods, /* tp_methods */ -}; - -PyObject * -PyAnextAwaitable_New(PyObject *awaitable, PyObject *default_value) -{ - anextawaitableobject *anext = PyObject_GC_New( - anextawaitableobject, &_PyAnextAwaitable_Type); - if (anext == NULL) { - return NULL; - } - anext->wrapped = Py_NewRef(awaitable); - anext->default_value = Py_NewRef(default_value); - _PyObject_GC_TRACK(anext); - return (PyObject *)anext; -} - - /* -------------------------------------- */ /* The asynchronous counterpart of calliterobject: the callable is called diff --git a/Objects/object.c b/Objects/object.c index a83f8d4c04ca079..e3f29b71301695e 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2522,7 +2522,6 @@ _PyObject_FiniState(PyInterpreterState *interp) extern PyTypeObject _PyACallIter_Type; extern PyTypeObject _PyACallIterAwaitable_Type; -extern PyTypeObject _PyAnextAwaitable_Type; extern PyTypeObject _PyLegacyEventHandler_Type; extern PyTypeObject _PyLineIterator; extern PyTypeObject _PyMemoryIter_Type; @@ -2617,7 +2616,6 @@ static PyTypeObject* static_types[_Py_NUM_MANAGED_PREINITIALIZED_TYPES] = { &Py_GenericAliasType, &_PyACallIter_Type, &_PyACallIterAwaitable_Type, - &_PyAnextAwaitable_Type, &_PyAsyncGenASend_Type, &_PyAsyncGenAThrow_Type, &_PyAsyncGenWrappedValue_Type, diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d28e6fa9cd01aed..2e3c307a100f6a8 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1960,52 +1960,6 @@ builtin_aiter_impl(PyObject *module, PyObject *object, PyObject *stop_value, return _PyACallIter_New(object, stop_value, stop_exception); } -PyObject *PyAnextAwaitable_New(PyObject *, PyObject *); - -/*[clinic input] -anext as builtin_anext - - async_iterator as aiterator: object - default: object = NULL - / - -Return the next item from the async iterator. - -If default is given and the async iterator is exhausted, -it is returned instead of raising StopAsyncIteration. -[clinic start generated code]*/ - -static PyObject * -builtin_anext_impl(PyObject *module, PyObject *aiterator, - PyObject *default_value) -/*[clinic end generated code: output=f02c060c163a81fa input=f3dc5a93f073e5ac]*/ -{ - PyTypeObject *t; - PyObject *awaitable; - - t = Py_TYPE(aiterator); - if (t->tp_as_async == NULL || t->tp_as_async->am_anext == NULL) { - PyErr_Format(PyExc_TypeError, - "'%.200s' object is not an async iterator", - t->tp_name); - return NULL; - } - - awaitable = (*t->tp_as_async->am_anext)(aiterator); - if (awaitable == NULL) { - return NULL; - } - if (default_value == NULL) { - return awaitable; - } - - PyObject* new_awaitable = PyAnextAwaitable_New( - awaitable, default_value); - Py_DECREF(awaitable); - return new_awaitable; -} - - /*[clinic input] len as builtin_len @@ -3500,7 +3454,6 @@ static PyMethodDef builtin_methods[] = { {"max", _PyCFunction_CAST(builtin_max), METH_FASTCALL | METH_KEYWORDS, max_doc}, {"min", _PyCFunction_CAST(builtin_min), METH_FASTCALL | METH_KEYWORDS, min_doc}, {"next", _PyCFunction_CAST(builtin_next), METH_FASTCALL, next_doc}, - BUILTIN_ANEXT_METHODDEF BUILTIN_OCT_METHODDEF BUILTIN_ORD_METHODDEF BUILTIN_POW_METHODDEF @@ -3539,6 +3492,111 @@ static struct PyModuleDef builtinsmodule = { }; +/* Builtins implemented in Python. + + Each entry below pairs the name of a builtin with the Python source that + defines it. _PyBuiltin_InitPythonFunctions() compiles and runs each + source in a shared private namespace and copies the named function into + the builtins dict. + +*/ + +static const char anext_source[] = + "_NOT_GIVEN = object()\n" + "\n" + "\n" + "def anext(async_iterator, default=_NOT_GIVEN, /):\n" + " \"\"\"Return the next item from the async iterator.\n" + "\n" + " If default is given and the async iterator is exhausted,\n" + " it is returned instead of raising StopAsyncIteration.\n" + " \"\"\"\n" + " cls = type(async_iterator)\n" + " try:\n" + " # Looked up on the type, like the C slot am_anext.\n" + " anext_method = cls.__anext__\n" + " except AttributeError:\n" + " raise TypeError(\n" + " f\"'{cls.__name__}' object is not an async iterator\"\n" + " ) from None\n" + " awaitable = anext_method(async_iterator)\n" + " if default is _NOT_GIVEN:\n" + " return awaitable\n" + " return _anext_with_default(awaitable, default)\n" + "\n" + "\n" + "async def _anext_with_default(awaitable, default):\n" + " try:\n" + " return await awaitable\n" + " except StopAsyncIteration:\n" + " return default\n"; + +struct builtin_python_function { + PyObject *name; + const char *source; +}; + +static const struct builtin_python_function builtin_python_functions[] = { + {&_Py_ID(anext), anext_source}, + {NULL, NULL}, +}; + +_Py_DECLARE_STR(anon_builtins, ""); + +int +_PyBuiltin_InitPythonFunctions(PyObject *dict) +{ + int rc = -1; + + PyObject *globals = PyDict_New(); + if (globals == NULL) { + return -1; + } + if (PyDict_SetItem(globals, &_Py_ID(__builtins__), dict) < 0) { + goto done; + } + if (PyDict_SetItem(globals, &_Py_ID(__name__), &_Py_ID(builtins)) < 0) { + goto done; + } + + for (const struct builtin_python_function *f = builtin_python_functions; + f->name != NULL; f++) + { + PyObject *code = Py_CompileStringObject(f->source, + &_Py_STR(anon_builtins), + Py_file_input, NULL, 0); + if (code == NULL) { + goto done; + } + PyObject *result = PyEval_EvalCode(code, globals, globals); + Py_DECREF(code); + if (result == NULL) { + goto done; + } + Py_DECREF(result); + + PyObject *func; + if (PyDict_GetItemRef(globals, f->name, &func) != 1) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_SystemError, + "builtin %U not defined by its Python source", + f->name); + } + goto done; + } + int r = PyDict_SetItem(dict, f->name, func); + Py_DECREF(func); + if (r < 0) { + goto done; + } + } + rc = 0; + +done: + Py_DECREF(globals); + return rc; +} + PyObject * _PyBuiltin_Init(PyInterpreterState *interp) { diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index c10bb03d8178161..5858ca9ff88ec22 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -1011,44 +1011,6 @@ builtin_aiter(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec return return_value; } -PyDoc_STRVAR(builtin_anext__doc__, -"anext($module, async_iterator, default=, /)\n" -"--\n" -"\n" -"Return the next item from the async iterator.\n" -"\n" -"If default is given and the async iterator is exhausted,\n" -"it is returned instead of raising StopAsyncIteration."); - -#define BUILTIN_ANEXT_METHODDEF \ - {"anext", _PyCFunction_CAST(builtin_anext), METH_FASTCALL, builtin_anext__doc__}, - -static PyObject * -builtin_anext_impl(PyObject *module, PyObject *aiterator, - PyObject *default_value); - -static PyObject * -builtin_anext(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *aiterator; - PyObject *default_value = NULL; - - if (!_PyArg_CheckPositional("anext", nargs, 1, 2)) { - goto exit; - } - aiterator = args[0]; - if (nargs < 2) { - goto skip_optional; - } - default_value = args[1]; -skip_optional: - return_value = builtin_anext_impl(module, aiterator, default_value); - -exit: - return return_value; -} - PyDoc_STRVAR(builtin_len__doc__, "len($module, obj, /)\n" "--\n" @@ -1539,4 +1501,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=5fb1ac6a4253ee2f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b56739f2e13f616a input=a9049054013a1b77]*/ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 500a1a1949a5a8a..b2ebcc67bee4544 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -924,6 +924,10 @@ pycore_init_builtins(PyThreadState *tstate) return _PyStatus_ERR("failed to add exceptions to builtins"); } + if (_PyBuiltin_InitPythonFunctions(builtins_dict) < 0) { + return _PyStatus_ERR("failed to add Python-implemented builtins"); + } + interp->builtins_copy = PyDict_Copy(interp->builtins); if (interp->builtins_copy == NULL) { goto error; diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index b8488899c4595de..67ced170243e4a2 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -60,7 +60,6 @@ Objects/iterobject.c - PyCallIter_Type - Objects/iterobject.c - PySeqIter_Type - Objects/iterobject.c - _PyACallIter_Type - Objects/iterobject.c - _PyACallIterAwaitable_Type - -Objects/iterobject.c - _PyAnextAwaitable_Type - Objects/lazyimportobject.c - PyLazyImport_Type - Objects/listobject.c - PyListIter_Type - Objects/listobject.c - PyListRevIter_Type - @@ -77,7 +76,6 @@ Objects/object.c - _PyNone_Type - Objects/object.c - _PyNotImplemented_Type - Objects/object.c - _PyACallIter_Type - Objects/object.c - _PyACallIterAwaitable_Type - -Objects/object.c - _PyAnextAwaitable_Type - Objects/odictobject.c - PyODictItems_Type - Objects/odictobject.c - PyODictIter_Type - Objects/odictobject.c - PyODictKeys_Type - From fcec7cfb55c6cd75ab8d0923ba7d6b24e5f0b09a Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 12 Sep 2026 16:38:03 +0530 Subject: [PATCH 2/3] use frozen modules instead --- .../pycore_global_objects_fini_generated.h | 2 - Include/internal/pycore_global_strings.h | 2 - .../internal/pycore_runtime_init_generated.h | 2 - .../internal/pycore_unicodeobject_generated.h | 8 -- Lib/_builtins.py | 42 +++++++ Lib/test/test_importlib/util.py | 3 +- Makefile.pre.in | 8 +- PCbuild/_freeze_module.vcxproj | 5 + PCbuild/_freeze_module.vcxproj.filters | 3 + Programs/_bootstrap_python.c | 2 + Python/bltinmodule.c | 106 +++++------------- Python/frozen.c | 2 + Python/pylifecycle.c | 10 +- Python/stdlib_module_names.h | 1 + Tools/build/freeze_modules.py | 3 + 15 files changed, 101 insertions(+), 98 deletions(-) create mode 100644 Lib/_builtins.py diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 3cf4e49b3b0bfa6..4fd6c618fb74404 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1310,7 +1310,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(bytes_characters)[253]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(bytes_characters)[254]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(bytes_characters)[255]); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_builtins)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_dictcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_genexpr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_lambda)); @@ -1588,7 +1587,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alphabet)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(anext)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(any)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arg)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 28350af3e80af8f..5b35c53e0aa03b1 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -30,7 +30,6 @@ extern "C" { /* The following is auto-generated by Tools/build/generate_global_objects.py. */ struct _Py_global_strings { struct { - STRUCT_FOR_STR(anon_builtins, "") STRUCT_FOR_STR(anon_dictcomp, "") STRUCT_FOR_STR(anon_genexpr, "") STRUCT_FOR_STR(anon_lambda, "") @@ -311,7 +310,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(all_threads) STRUCT_FOR_ID(allow_code) STRUCT_FOR_ID(alphabet) - STRUCT_FOR_ID(anext) STRUCT_FOR_ID(any) STRUCT_FOR_ID(append) STRUCT_FOR_ID(arg) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 43e6b974d419a29..c80925f020186ba 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1305,7 +1305,6 @@ extern "C" { } #define _Py_str_literals_INIT { \ - INIT_STR(anon_builtins, ""), \ INIT_STR(anon_dictcomp, ""), \ INIT_STR(anon_genexpr, ""), \ INIT_STR(anon_lambda, ""), \ @@ -1586,7 +1585,6 @@ extern "C" { INIT_ID(all_threads), \ INIT_ID(allow_code), \ INIT_ID(alphabet), \ - INIT_ID(anext), \ INIT_ID(any), \ INIT_ID(append), \ INIT_ID(arg), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index d057702eef4cb06..b30cfc678de1cd7 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1020,10 +1020,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(anext); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(any); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3480,10 +3476,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_STR(anon_builtins); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_dictcomp); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/_builtins.py b/Lib/_builtins.py new file mode 100644 index 000000000000000..4642a69b3aaf22c --- /dev/null +++ b/Lib/_builtins.py @@ -0,0 +1,42 @@ +"""Builtins implemented in Python. + +This module is frozen into the interpreter and imported during startup, +before the import system exists. The names listed in ``__all__`` are +copied into the ``builtins`` module. +""" + +__all__ = ['anext'] + +_NOT_GIVEN = object() + + +def anext(async_iterator, default=_NOT_GIVEN, /): + """Return the next item from the async iterator. + + If default is given and the async iterator is exhausted, + it is returned instead of raising StopAsyncIteration. + """ + cls = type(async_iterator) + try: + # Looked up on the type, like the C slot am_anext. + anext_method = cls.__anext__ + except AttributeError: + raise TypeError( + f"'{cls.__name__}' object is not an async iterator" + ) from None + awaitable = anext_method(async_iterator) + if default is _NOT_GIVEN: + return awaitable + return _anext_with_default(awaitable, default) + + +async def _anext_with_default(awaitable, default): + try: + return await awaitable + except StopAsyncIteration: + return default + + +for _name in __all__: + globals()[_name].__module__ = 'builtins' +del _name diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index 6399f952f9e912b..4269940816ccafc 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -69,7 +69,8 @@ def import_importlib(module_name): fresh = ('importlib',) if '.' in module_name else () frozen = import_helper.import_fresh_module(module_name) source = import_helper.import_fresh_module(module_name, fresh=fresh, - blocked=('_frozen_importlib', '_frozen_importlib_external')) + blocked=('_frozen_importlib', '_frozen_importlib_external', + '_builtins')) return {'Frozen': frozen, 'Source': source} diff --git a/Makefile.pre.in b/Makefile.pre.in index 78a486623181fa8..9d5deb3902fd365 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1623,7 +1623,8 @@ Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS) BOOTSTRAP_HEADERS = \ Python/frozen_modules/importlib._bootstrap.h \ Python/frozen_modules/importlib._bootstrap_external.h \ - Python/frozen_modules/zipimport.h + Python/frozen_modules/zipimport.h \ + Python/frozen_modules/_builtins.h Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) $(PYTHON_HEADERS) @@ -1664,6 +1665,7 @@ FROZEN_FILES_IN = \ Lib/importlib/_bootstrap.py \ Lib/importlib/_bootstrap_external.py \ Lib/zipimport.py \ + Lib/_builtins.py \ Lib/abc.py \ Lib/codecs.py \ Lib/io.py \ @@ -1690,6 +1692,7 @@ FROZEN_FILES_OUT = \ Python/frozen_modules/importlib._bootstrap.h \ Python/frozen_modules/importlib._bootstrap_external.h \ Python/frozen_modules/zipimport.h \ + Python/frozen_modules/_builtins.h \ Python/frozen_modules/abc.h \ Python/frozen_modules/codecs.h \ Python/frozen_modules/io.h \ @@ -1735,6 +1738,9 @@ Python/frozen_modules/importlib._bootstrap_external.h: Lib/importlib/_bootstrap_ Python/frozen_modules/zipimport.h: Lib/zipimport.py $(FREEZE_MODULE_BOOTSTRAP_DEPS) $(FREEZE_MODULE_BOOTSTRAP) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h +Python/frozen_modules/_builtins.h: Lib/_builtins.py $(FREEZE_MODULE_BOOTSTRAP_DEPS) + $(FREEZE_MODULE_BOOTSTRAP) _builtins $(srcdir)/Lib/_builtins.py Python/frozen_modules/_builtins.h + Python/frozen_modules/abc.h: Lib/abc.py $(FREEZE_MODULE_DEPS) $(FREEZE_MODULE) abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 469fd77cc8be9dc..94eaee0f7c0dd18 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -306,6 +306,11 @@ $(IntDir)zipimport.g.h $(GeneratedFrozenModulesDir)Python\frozen_modules\zipimport.h + + _builtins + $(IntDir)_builtins.g.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\_builtins.h + abc $(IntDir)abc.g.h diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 976c99b7d24bdfd..bc3f4df5d874ab8 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -549,6 +549,9 @@ Python Files + + Python Files + Python Files diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c index 6443d814a22dabf..5509e9777117095 100644 --- a/Programs/_bootstrap_python.c +++ b/Programs/_bootstrap_python.c @@ -13,6 +13,7 @@ #include "Python/frozen_modules/importlib._bootstrap.h" #include "Python/frozen_modules/importlib._bootstrap_external.h" #include "Python/frozen_modules/zipimport.h" +#include "Python/frozen_modules/_builtins.h" /* End includes */ /* Note that a negative size indicates a package. */ @@ -21,6 +22,7 @@ static const struct _frozen bootstrap_modules[] = { {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap)}, {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external)}, {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)}, + {"_builtins", _Py_M___builtins, (int)sizeof(_Py_M___builtins)}, {0, 0, 0} /* bootstrap sentinel */ }; static const struct _frozen stdlib_modules[] = { diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 2e3c307a100f6a8..9ef1dd5e9980a58 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3494,97 +3494,42 @@ static struct PyModuleDef builtinsmodule = { /* Builtins implemented in Python. - Each entry below pairs the name of a builtin with the Python source that - defines it. _PyBuiltin_InitPythonFunctions() compiles and runs each - source in a shared private namespace and copies the named function into - the builtins dict. - -*/ - -static const char anext_source[] = - "_NOT_GIVEN = object()\n" - "\n" - "\n" - "def anext(async_iterator, default=_NOT_GIVEN, /):\n" - " \"\"\"Return the next item from the async iterator.\n" - "\n" - " If default is given and the async iterator is exhausted,\n" - " it is returned instead of raising StopAsyncIteration.\n" - " \"\"\"\n" - " cls = type(async_iterator)\n" - " try:\n" - " # Looked up on the type, like the C slot am_anext.\n" - " anext_method = cls.__anext__\n" - " except AttributeError:\n" - " raise TypeError(\n" - " f\"'{cls.__name__}' object is not an async iterator\"\n" - " ) from None\n" - " awaitable = anext_method(async_iterator)\n" - " if default is _NOT_GIVEN:\n" - " return awaitable\n" - " return _anext_with_default(awaitable, default)\n" - "\n" - "\n" - "async def _anext_with_default(awaitable, default):\n" - " try:\n" - " return await awaitable\n" - " except StopAsyncIteration:\n" - " return default\n"; - -struct builtin_python_function { - PyObject *name; - const char *source; -}; - -static const struct builtin_python_function builtin_python_functions[] = { - {&_Py_ID(anext), anext_source}, - {NULL, NULL}, -}; - -_Py_DECLARE_STR(anon_builtins, ""); + Lib/_builtins.py is frozen into the interpreter as a bootstrap module + (see Tools/build/freeze_modules.py), so it can be imported here before + the import system exists. The names in its __all__ are copied into the + builtins dict. */ int _PyBuiltin_InitPythonFunctions(PyObject *dict) { - int rc = -1; - - PyObject *globals = PyDict_New(); - if (globals == NULL) { + if (PyImport_ImportFrozenModule("_builtins") <= 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, + "frozen module _builtins not found"); + } + return -1; + } + PyObject *mod = PyImport_AddModuleRef("_builtins"); + if (mod == NULL) { return -1; } - if (PyDict_SetItem(globals, &_Py_ID(__builtins__), dict) < 0) { + + int rc = -1; + PyObject *all = PyObject_GetAttr(mod, &_Py_ID(__all__)); + if (all == NULL) { goto done; } - if (PyDict_SetItem(globals, &_Py_ID(__name__), &_Py_ID(builtins)) < 0) { + Py_ssize_t n = PyList_Size(all); + if (n < 0) { goto done; } - - for (const struct builtin_python_function *f = builtin_python_functions; - f->name != NULL; f++) - { - PyObject *code = Py_CompileStringObject(f->source, - &_Py_STR(anon_builtins), - Py_file_input, NULL, 0); - if (code == NULL) { - goto done; - } - PyObject *result = PyEval_EvalCode(code, globals, globals); - Py_DECREF(code); - if (result == NULL) { + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *name = PyList_GET_ITEM(all, i); + PyObject *func = PyObject_GetAttr(mod, name); + if (func == NULL) { goto done; } - Py_DECREF(result); - - PyObject *func; - if (PyDict_GetItemRef(globals, f->name, &func) != 1) { - if (!PyErr_Occurred()) { - PyErr_Format(PyExc_SystemError, - "builtin %U not defined by its Python source", - f->name); - } - goto done; - } - int r = PyDict_SetItem(dict, f->name, func); + int r = PyDict_SetItem(dict, name, func); Py_DECREF(func); if (r < 0) { goto done; @@ -3593,7 +3538,8 @@ _PyBuiltin_InitPythonFunctions(PyObject *dict) rc = 0; done: - Py_DECREF(globals); + Py_XDECREF(all); + Py_DECREF(mod); return rc; } diff --git a/Python/frozen.c b/Python/frozen.c index 9433d90c15e2eca..b1fd0fb427298b9 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -44,6 +44,7 @@ #include "frozen_modules/importlib._bootstrap.h" #include "frozen_modules/importlib._bootstrap_external.h" #include "frozen_modules/zipimport.h" +#include "frozen_modules/_builtins.h" #include "frozen_modules/abc.h" #include "frozen_modules/codecs.h" #include "frozen_modules/io.h" @@ -71,6 +72,7 @@ static const struct _frozen bootstrap_modules[] = { {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap), false}, {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external), false}, {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport), false}, + {"_builtins", _Py_M___builtins, (int)sizeof(_Py_M___builtins), false}, {0, 0, 0} /* bootstrap sentinel */ }; static const struct _frozen stdlib_modules[] = { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index b2ebcc67bee4544..e4b0258706714fa 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -924,8 +924,14 @@ pycore_init_builtins(PyThreadState *tstate) return _PyStatus_ERR("failed to add exceptions to builtins"); } - if (_PyBuiltin_InitPythonFunctions(builtins_dict) < 0) { - return _PyStatus_ERR("failed to add Python-implemented builtins"); + /* The Python-implemented builtins live in the frozen _builtins module. + Programs/_freeze_module has no frozen modules (it's what creates + them) and opts out via _install_importlib, like the import system. */ + const PyConfig *config = _PyInterpreterState_GetConfig(interp); + if (config->_install_importlib) { + if (_PyBuiltin_InitPythonFunctions(builtins_dict) < 0) { + return _PyStatus_ERR("failed to add Python-implemented builtins"); + } } interp->builtins_copy = PyDict_Copy(interp->builtins); diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 8937e666bbbdd5b..bd5db458c0e89f4 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -12,6 +12,7 @@ static const char* _Py_stdlib_module_names[] = { "_asyncio", "_bisect", "_blake2", +"_builtins", "_bz2", "_codecs", "_codecs_cn", diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py index a866336fa78879e..5cfa6141fd41db9 100644 --- a/Tools/build/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -45,6 +45,8 @@ # This module is important because some Python builds rely # on a builtin zip file instead of a filesystem. 'zipimport', + # Builtins implemented in Python; loaded while builtins is set up. + '_builtins', ]), # (You can delete entries from here down to the end of the list.) ('stdlib - startup, without site (python -S)', [ @@ -91,6 +93,7 @@ 'importlib._bootstrap', 'importlib._bootstrap_external', 'zipimport', + '_builtins', } From 0903f033016568a13d9121abf40081c60009cbe6 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 12 Sep 2026 16:40:52 +0530 Subject: [PATCH 3/3] add news --- .../2026-09-12-16-40-00.gh-issue-157361.anextpy.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-12-16-40-00.gh-issue-157361.anextpy.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-12-16-40-00.gh-issue-157361.anextpy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-12-16-40-00.gh-issue-157361.anextpy.rst new file mode 100644 index 000000000000000..c78e75b723e0d13 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-12-16-40-00.gh-issue-157361.anextpy.rst @@ -0,0 +1,4 @@ +Implement :func:`anext` in Python instead of C, in a frozen ``_builtins`` +module. The awaitable returned by ``anext(it, default)`` is now a plain +coroutine, so introspection tools such as :func:`asyncio.print_call_graph` +can see through it into :meth:`~object.__anext__`.