diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index a086c60905402e..bfd39507461d03 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -85,6 +85,10 @@ See also :ref:`Reflection `. .. versionadded:: 3.11 + .. versionchanged:: 3.16 + May return ``NULL`` for a coroutine frame before its first suspension; + see :attr:`frame.f_generator`. + .. c:function:: PyObject* PyFrame_GetGlobals(PyFrameObject *frame) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 16c2dab5b8adba..0876525461d747 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -834,6 +834,25 @@ not have to be) the original ``STACK[-2]``. Previously, this instruction did not have an oparg. +.. opcode:: END_AWAITED_CALL (delta) + + Follows a call whose result is awaited right away (``await f()``). + The compiler pushes an extra ``NULL``, the *lazy outcome*, below the + callable of such a call, and the call may run the body of a coroutine + function without creating the coroutine object, reporting in the lazy + outcome how the body ended. + + If the lazy outcome is ``NULL``, it is removed and execution continues with + :opcode:`GET_AWAITABLE`. If the body returned, it is removed and + execution continues after the :opcode:`END_SEND` of the following + :opcode:`SEND` loop. If the body suspended, the coroutine object is + created and the bytecode counter is incremented by *delta*, to the + :opcode:`YIELD_VALUE` of that loop, with ``STACK[-3:]`` set to the + coroutine, ``NULL`` and the yielded value. + + .. versionadded:: 3.16 + + .. opcode:: GET_AITER Implements ``STACK[-1] = STACK[-1].__aiter__()``. @@ -1615,7 +1634,8 @@ iterations of the loop. * ``self`` or ``NULL`` * The remaining positional arguments - ``argc`` is the total of the positional arguments, excluding ``self``. + ``argc >> 1`` is the total of the positional arguments, excluding ``self``. + The low bit of ``argc`` is set when the result is awaited right away. ``CALL`` pops all arguments and the callable object off the stack, calls the callable object with those arguments, and pushes the return value @@ -1629,6 +1649,10 @@ iterations of the loop. .. versionchanged:: 3.13 Calls with keyword arguments are now handled by :opcode:`CALL_KW`. + .. versionchanged:: next + The number of arguments is now ``argc >> 1``; the low bit marks an + awaited call. + .. opcode:: CALL_KW (argc) @@ -1641,7 +1665,8 @@ iterations of the loop. * The named arguments * A :class:`tuple` of keyword argument names - ``argc`` is the total of the positional and named arguments, excluding ``self``. + ``argc >> 1`` is the total of the positional and named arguments, excluding + ``self``. The low bit of ``argc`` is set when the result is awaited right away. The length of the tuple of keyword argument names is the number of named arguments. ``CALL_KW`` pops all arguments, the keyword names, and the callable object @@ -1650,21 +1675,29 @@ iterations of the loop. .. versionadded:: 3.13 + .. versionchanged:: next + The number of arguments is now ``argc >> 1``; the low bit marks an + awaited call. + .. opcode:: CALL_FUNCTION_EX (flags) Calls a callable object with variable set of positional and keyword - arguments. If the lowest bit of *flags* is set, the top of the stack - contains a mapping object containing additional keyword arguments. + arguments. The top of the stack contains a mapping object with additional + keyword arguments, or ``NULL``. Before the callable is called, the mapping object and iterable object are each "unpacked" and their contents passed in as keyword and positional arguments respectively. ``CALL_FUNCTION_EX`` pops all arguments and the callable object off the stack, calls the callable object with those arguments, and pushes the return value returned by the callable object. + The lowest bit of *flags* is set when the result is awaited right away. .. versionadded:: 3.6 + .. versionchanged:: next + The lowest bit of *flags* marks an awaited call. + .. opcode:: PUSH_NULL diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 51c465e8f00b5e..328bfea8a372ab 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -151,7 +151,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | coroutine object that | | | | owns this frame, or | | | | ``None`` if the frame is | -| | | of a regular function | +| | | of a regular function or | +| | | the generator is not | +| | | materialized yet | +-----------------+-------------------+---------------------------+ | | f_trace | tracing function for this | | | | frame, or ``None`` | diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 02663ed465f424..07fa9cf5441e64 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1736,10 +1736,22 @@ Special read-only attributes * - .. attribute:: frame.f_generator - The :term:`generator` or :term:`coroutine` object that owns this frame, - or ``None`` if the frame is a normal function. + or ``None`` if the frame is a normal function or if the generator + is not materialized yet. + + .. impl-detail:: + + A coroutine that is awaited as soon as it is called + (``await coro_func()``) may run without a coroutine object until + it first suspends, so this attribute is ``None`` before that. + Use :attr:`~codeobject.co_flags` of :attr:`frame.f_code` to tell + coroutine frames apart from normal function frames. .. versionadded:: 3.14 + .. versionchanged:: 3.16 + May be ``None`` for a coroutine frame before its first suspension. + .. index:: single: f_trace (frame attribute) single: f_trace_lines (frame attribute) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 33328b2ee00521..a86f9053ff6225 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1147,6 +1147,19 @@ datetime and time (Contributed by Paul Ganssle in :gh:`158252`, based on work by Stan Ulbrych.) +Interpreter +----------- + +* A coroutine that is awaited as soon as it is called + (``await coro_func()``) now runs without creating a coroutine object + unless it suspends, making awaits that complete immediately about twice + as fast in a micro-benchmark (``await noop()``: 52 ns to 27 ns on x86-64). + This does not apply while coroutine origin tracking is enabled or inside + code compiled by the experimental JIT. + Until it first suspends, :attr:`frame.f_generator` is ``None`` for such a + coroutine; see :ref:`whatsnew316-porting`. + (Contributed by Andrew Svetlov in :gh:`161170`.) + re -- @@ -1405,12 +1418,23 @@ New deprecations .. include:: ../deprecations/pending-removal-in-future.rst +.. _whatsnew316-porting: + Porting to Python 3.16 ====================== This section lists previously described changes and other bugfixes that may require changes to your code. +* :attr:`frame.f_generator` and :c:func:`PyFrame_GetGenerator` may now + return ``None``/``NULL`` for a coroutine frame before its first + suspension, because a coroutine that is awaited as soon as it is called + (``await coro_func()``) runs without a coroutine object until then. + Code that used ``frame.f_generator is None`` to detect frames of normal + functions should check ``frame.f_code.co_flags`` for + :data:`~inspect.CO_COROUTINE` instead. + (Contributed by Andrew Svetlov in :gh:`161170`.) + * In :mod:`tkinter`, the *name* parameter of the :meth:`~tkinter.Misc.wait_variable`, :meth:`~tkinter.Misc.setvar` and :meth:`~tkinter.Misc.getvar` methods and the *value* parameter of @@ -1488,6 +1512,22 @@ that may require changes to your code. (Contributed by An Long in :gh:`86768`.) +CPython bytecode changes +======================== + +* The oparg of :opcode:`CALL` and :opcode:`CALL_KW` is now the number of + arguments shifted left by one. The low bit of their oparg, and of the + oparg of :opcode:`CALL_FUNCTION_EX`, marks a call whose result is awaited + right away. + (Contributed by Andrew Svetlov in :gh:`161170`.) + +* ``await `` now pushes an extra ``NULL`` below the callable and + follows the call with the new :opcode:`END_AWAITED_CALL` instruction, + which either goes on to :opcode:`GET_AWAITABLE`, skips the + :opcode:`SEND` loop, or jumps to its :opcode:`YIELD_VALUE`. + (Contributed by Andrew Svetlov in :gh:`161170`.) + + Build changes ============= diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a0fea389b6502c..9a1f6ec2221e46 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -267,6 +267,12 @@ extern int _PyEval_EnsureBuiltinsWithModule( PyObject **p_builtins); PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); +// Lazily created coroutines (see RETURN_GENERATOR in bytecodes.c). +// Exported for _testinternalcapi and external JIT support. +PyAPI_FUNC(PyObject *) _PyCoro_MaterializeLazy(_PyInterpreterFrame *frame, + _PyInterpreterFrame *caller); +PyAPI_FUNC(PyGenObject *) _PyEval_MaterializeLazyCoro(PyThreadState *tstate, + _PyInterpreterFrame *frame); /* Handle signals, pending calls, GIL drop request and asynchronous exception */ diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index a85fa9bd32853c..ea280a324e758a 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -152,6 +152,8 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * dest->stackpointer = dest->localsplus + stacktop; // visited is GC bookkeeping for the current stack walk, not frame state. dest->visited = 0; + // A lazily started coroutine stops being lazy once its frame is copied. + dest->lazy_coro = 0; #ifdef Py_DEBUG dest->stackpointer_valid = src->stackpointer_valid; dest->lltrace = src->lltrace; @@ -208,6 +210,7 @@ _PyFrame_Initialize( frame->return_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; + frame->lazy_coro = 0; #ifdef Py_DEBUG frame->stackpointer_valid = 1; frame->lltrace = 0; @@ -441,6 +444,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int #endif frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; + frame->lazy_coro = 0; #ifdef Py_DEBUG frame->stackpointer_valid = 1; frame->lltrace = 0; diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 0d6a37cc2c1a17..4937e905081fae 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -44,9 +44,10 @@ struct _PyInterpreterFrame { char owner; uint8_t visited:1; uint8_t external:1; /* Set when the frame was pushed by something other than the VM */ + uint8_t lazy_coro:1; #ifdef Py_DEBUG uint8_t stackpointer_valid:1; - uint8_t lltrace:5; + uint8_t lltrace:4; #endif /* Locals and stack */ _PyStackRef localsplus[1]; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 15d3ad6f3a336b..ddc005a9b0f2f2 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -304,6 +304,7 @@ Known values: Python 3.16a1 3704 (Replace DELETE_ATTR with PUSH_NULL; STORE_ATTR) Python 3.16a1 3705 (Add INTRINSIC_ADD_CONDITIONAL_ANNOTATION) Python 3.16a1 3706 (Add INTRINSIC_MATCH_CLASS_ISINSTANCE) + Python 3.16a1 3707 (Lazily materialize coroutine objects for awaited calls) Python 3.17 will start with 3750 @@ -313,7 +314,7 @@ Known values: */ -#define PYC_MAGIC_NUMBER 3706 +#define PYC_MAGIC_NUMBER 3707 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 04b826ed09144e..a48cbff3c92f45 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -93,13 +93,17 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case CACHE: return 0; case CALL: - return 2 + oparg; + return 2 + (oparg & ~CALL_AWAITED); case CALL_ALLOC_AND_ENTER_INIT: return 2 + oparg; case CALL_BOUND_METHOD_EXACT_ARGS: return 2 + oparg; + case CALL_BOUND_METHOD_EXACT_ARGS_AWAITED: + return 2 + oparg; case CALL_BOUND_METHOD_GENERAL: return 2 + oparg; + case CALL_BOUND_METHOD_GENERAL_AWAITED: + return 2 + oparg; case CALL_BUILTIN_CLASS: return 2 + oparg; case CALL_BUILTIN_FAST: @@ -121,13 +125,17 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case CALL_ISINSTANCE: return 4; case CALL_KW: - return 3 + oparg; + return 3 + (oparg & ~CALL_AWAITED); case CALL_KW_BOUND_METHOD: return 3 + oparg; + case CALL_KW_BOUND_METHOD_AWAITED: + return 3 + oparg; case CALL_KW_NON_PY: return 3 + oparg; case CALL_KW_PY: return 3 + oparg; + case CALL_KW_PY_AWAITED: + return 3 + oparg; case CALL_LEN: return 3; case CALL_LIST_APPEND: @@ -144,8 +152,12 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_PY_EXACT_ARGS: return 2 + oparg; + case CALL_PY_EXACT_ARGS_AWAITED: + return 2 + oparg; case CALL_PY_GENERAL: return 2 + oparg; + case CALL_PY_GENERAL_AWAITED: + return 2 + oparg; case CALL_STR_1: return 3; case CALL_TUPLE_1: @@ -190,6 +202,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + (oparg - 1); case END_ASYNC_FOR: return 2; + case END_AWAITED_CALL: + return 2; case END_FOR: return 1; case END_SEND: @@ -235,11 +249,11 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case IMPORT_NAME: return 2; case INSTRUMENTED_CALL: - return 2 + oparg; + return 2 + (oparg & ~CALL_AWAITED); case INSTRUMENTED_CALL_FUNCTION_EX: return 4; case INSTRUMENTED_CALL_KW: - return 3 + oparg; + return 3 + (oparg & ~CALL_AWAITED); case INSTRUMENTED_END_ASYNC_FOR: return 2; case INSTRUMENTED_END_FOR: @@ -593,8 +607,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case CALL_BOUND_METHOD_EXACT_ARGS: return 0; + case CALL_BOUND_METHOD_EXACT_ARGS_AWAITED: + return 0; case CALL_BOUND_METHOD_GENERAL: return 0; + case CALL_BOUND_METHOD_GENERAL_AWAITED: + return 0; case CALL_BUILTIN_CLASS: return 1; case CALL_BUILTIN_FAST: @@ -619,10 +637,14 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CALL_KW_BOUND_METHOD: return 0; + case CALL_KW_BOUND_METHOD_AWAITED: + return 0; case CALL_KW_NON_PY: return 1; case CALL_KW_PY: return 0; + case CALL_KW_PY_AWAITED: + return 0; case CALL_LEN: return 1; case CALL_LIST_APPEND: @@ -639,8 +661,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CALL_PY_EXACT_ARGS: return 0; + case CALL_PY_EXACT_ARGS_AWAITED: + return 0; case CALL_PY_GENERAL: return 0; + case CALL_PY_GENERAL_AWAITED: + return 0; case CALL_STR_1: return 1; case CALL_TUPLE_1: @@ -685,6 +711,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1 + (oparg - 1); case END_ASYNC_FOR: return 0; + case END_AWAITED_CALL: + return 1; case END_FOR: return 0; case END_SEND: @@ -1130,21 +1158,25 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS_AWAITED] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BOUND_METHOD_GENERAL_AWAITED] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_EX_NON_PY_GENERAL] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_EX_PY] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_KW_BOUND_METHOD_AWAITED] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_KW_PY_AWAITED] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1153,7 +1185,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_PY_EXACT_ARGS_AWAITED] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_PY_GENERAL_AWAITED] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, @@ -1176,6 +1210,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [END_AWAITED_CALL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1199,7 +1234,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, @@ -1345,7 +1380,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { }; #endif -#define MAX_UOP_PER_EXPANSION 11 +#define MAX_UOP_PER_EXPANSION 13 struct opcode_macro_expansion { int nuops; struct { int16_t uop; int8_t size; int8_t offset; } uops[MAX_UOP_PER_EXPANSION]; @@ -1383,19 +1418,23 @@ _PyOpcode_macro_expansion[256] = { [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, OPARG_SIMPLE, 0 } } }, [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_OBJECT, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _ALLOCATE_OBJECT, OPARG_SIMPLE, 3 }, { _CREATE_INIT_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 11, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS_AWAITED] = { .nuops = 13, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CLEAR_CALL_AWAITED, OPARG_REPLACED, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _START_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_BOUND_METHOD_GENERAL] = { .nuops = 8, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_BOUND_METHOD_GENERAL_AWAITED] = { .nuops = 10, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CLEAR_CALL_AWAITED, OPARG_REPLACED, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _START_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_BUILTIN_CLASS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_BUILTIN_FAST] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_BUILTIN_O] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_EX_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CALL_FUNCTION_EX_NON_PY_GENERAL, OPARG_SIMPLE, 1 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 1 } } }, - [CALL_EX_PY] = { .nuops = 7, .uops = { { _RECORD_4OS, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CHECK_IS_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _PY_FRAME_EX, OPARG_SIMPLE, 1 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [CALL_EX_PY] = { .nuops = 8, .uops = { { _RECORD_4OS, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CHECK_IS_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _PY_FRAME_EX, OPARG_SIMPLE, 1 }, { _MAYBE_LAZY_CORO, OPARG_REPLACED, 1 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, [CALL_INTRINSIC_1] = { .nuops = 2, .uops = { { _CALL_INTRINSIC_1, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 3, .uops = { { _CALL_INTRINSIC_2, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 3, .uops = { { _GUARD_THIRD_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_ISINSTANCE, OPARG_SIMPLE, 3 }, { _CALL_ISINSTANCE, OPARG_SIMPLE, 3 } } }, [CALL_KW_BOUND_METHOD] = { .nuops = 7, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_KW_BOUND_METHOD_AWAITED] = { .nuops = 9, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CLEAR_CALL_AWAITED, OPARG_REPLACED, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _START_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_KW_PY] = { .nuops = 7, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_KW_PY_AWAITED] = { .nuops = 9, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CLEAR_CALL_AWAITED, OPARG_REPLACED, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _START_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_LEN] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [CALL_LIST_APPEND] = { .nuops = 6, .uops = { { _GUARD_CALLABLE_LIST_APPEND, OPARG_SIMPLE, 3 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 3 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 3 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 5, .uops = { { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, @@ -1404,7 +1443,9 @@ _PyOpcode_macro_expansion[256] = { [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_PY_EXACT_ARGS] = { .nuops = 9, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_PY_EXACT_ARGS_AWAITED] = { .nuops = 11, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CLEAR_CALL_AWAITED, OPARG_REPLACED, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _START_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_PY_GENERAL] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_PY_GENERAL_AWAITED] = { .nuops = 9, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CLEAR_CALL_AWAITED, OPARG_REPLACED, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _START_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_STR_1] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_STR_1, OPARG_SIMPLE, 3 }, { _CALL_STR_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_TUPLE_1] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_TYPE_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TYPE_1, OPARG_SIMPLE, 3 }, { _CALL_TYPE_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, @@ -1425,6 +1466,7 @@ _PyOpcode_macro_expansion[256] = { [DELETE_SUBSCR] = { .nuops = 1, .uops = { { _DELETE_SUBSCR, OPARG_SIMPLE, 0 } } }, [DICT_MERGE] = { .nuops = 2, .uops = { { _DICT_MERGE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [DICT_UPDATE] = { .nuops = 2, .uops = { { _DICT_UPDATE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [END_AWAITED_CALL] = { .nuops = 2, .uops = { { _END_AWAITED_CALL, OPARG_REPLACED, 0 }, { _POP_LAZY_OUTCOME, OPARG_SIMPLE, 0 } } }, [END_FOR] = { .nuops = 1, .uops = { { _END_FOR, OPARG_SIMPLE, 0 } } }, [END_SEND] = { .nuops = 1, .uops = { { _END_SEND, OPARG_SIMPLE, 0 } } }, [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { _EXIT_INIT_CHECK, OPARG_SIMPLE, 0 } } }, @@ -1545,7 +1587,7 @@ _PyOpcode_macro_expansion[256] = { [UNPACK_SEQUENCE_TUPLE] = { .nuops = 2, .uops = { { _GUARD_TOS_TUPLE, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_TUPLE, OPARG_SIMPLE, 1 } } }, [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 2, .uops = { { _GUARD_TOS_TUPLE, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_TWO_TUPLE, OPARG_SIMPLE, 1 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { _WITH_EXCEPT_START, OPARG_SIMPLE, 0 } } }, - [YIELD_VALUE] = { .nuops = 2, .uops = { { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _YIELD_VALUE, OPARG_SIMPLE, 0 } } }, + [YIELD_VALUE] = { .nuops = 3, .uops = { { _CHECK_LAZY_CORO, OPARG_REPLACED, 0 }, { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _YIELD_VALUE, OPARG_SIMPLE, 0 } } }, }; #endif // NEED_OPCODE_METADATA @@ -1583,7 +1625,9 @@ const char *_PyOpcode_OpName[267] = { [CALL] = "CALL", [CALL_ALLOC_AND_ENTER_INIT] = "CALL_ALLOC_AND_ENTER_INIT", [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [CALL_BOUND_METHOD_EXACT_ARGS_AWAITED] = "CALL_BOUND_METHOD_EXACT_ARGS_AWAITED", [CALL_BOUND_METHOD_GENERAL] = "CALL_BOUND_METHOD_GENERAL", + [CALL_BOUND_METHOD_GENERAL_AWAITED] = "CALL_BOUND_METHOD_GENERAL_AWAITED", [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [CALL_BUILTIN_FAST] = "CALL_BUILTIN_FAST", [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", @@ -1596,8 +1640,10 @@ const char *_PyOpcode_OpName[267] = { [CALL_ISINSTANCE] = "CALL_ISINSTANCE", [CALL_KW] = "CALL_KW", [CALL_KW_BOUND_METHOD] = "CALL_KW_BOUND_METHOD", + [CALL_KW_BOUND_METHOD_AWAITED] = "CALL_KW_BOUND_METHOD_AWAITED", [CALL_KW_NON_PY] = "CALL_KW_NON_PY", [CALL_KW_PY] = "CALL_KW_PY", + [CALL_KW_PY_AWAITED] = "CALL_KW_PY_AWAITED", [CALL_LEN] = "CALL_LEN", [CALL_LIST_APPEND] = "CALL_LIST_APPEND", [CALL_METHOD_DESCRIPTOR_FAST] = "CALL_METHOD_DESCRIPTOR_FAST", @@ -1606,7 +1652,9 @@ const char *_PyOpcode_OpName[267] = { [CALL_METHOD_DESCRIPTOR_O] = "CALL_METHOD_DESCRIPTOR_O", [CALL_NON_PY_GENERAL] = "CALL_NON_PY_GENERAL", [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", + [CALL_PY_EXACT_ARGS_AWAITED] = "CALL_PY_EXACT_ARGS_AWAITED", [CALL_PY_GENERAL] = "CALL_PY_GENERAL", + [CALL_PY_GENERAL_AWAITED] = "CALL_PY_GENERAL_AWAITED", [CALL_STR_1] = "CALL_STR_1", [CALL_TUPLE_1] = "CALL_TUPLE_1", [CALL_TYPE_1] = "CALL_TYPE_1", @@ -1629,6 +1677,7 @@ const char *_PyOpcode_OpName[267] = { [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", [END_ASYNC_FOR] = "END_ASYNC_FOR", + [END_AWAITED_CALL] = "END_AWAITED_CALL", [END_FOR] = "END_FOR", [END_SEND] = "END_SEND", [ENTER_EXECUTOR] = "ENTER_EXECUTOR", @@ -1828,7 +1877,6 @@ const uint8_t _PyOpcode_Caches[256] = { PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { - [117] = 117, [118] = 118, [119] = 119, [120] = 120, @@ -1839,12 +1887,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [125] = 125, [126] = 126, [127] = 127, - [219] = 219, - [220] = 220, - [221] = 221, - [222] = 222, - [223] = 223, - [224] = 224, [225] = 225, [226] = 226, [227] = 227, @@ -1883,7 +1925,9 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL] = CALL, [CALL_ALLOC_AND_ENTER_INIT] = CALL, [CALL_BOUND_METHOD_EXACT_ARGS] = CALL, + [CALL_BOUND_METHOD_EXACT_ARGS_AWAITED] = CALL, [CALL_BOUND_METHOD_GENERAL] = CALL, + [CALL_BOUND_METHOD_GENERAL_AWAITED] = CALL, [CALL_BUILTIN_CLASS] = CALL, [CALL_BUILTIN_FAST] = CALL, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, @@ -1896,8 +1940,10 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_ISINSTANCE] = CALL, [CALL_KW] = CALL_KW, [CALL_KW_BOUND_METHOD] = CALL_KW, + [CALL_KW_BOUND_METHOD_AWAITED] = CALL_KW, [CALL_KW_NON_PY] = CALL_KW, [CALL_KW_PY] = CALL_KW, + [CALL_KW_PY_AWAITED] = CALL_KW, [CALL_LEN] = CALL, [CALL_LIST_APPEND] = CALL, [CALL_METHOD_DESCRIPTOR_FAST] = CALL, @@ -1906,7 +1952,9 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_METHOD_DESCRIPTOR_O] = CALL, [CALL_NON_PY_GENERAL] = CALL, [CALL_PY_EXACT_ARGS] = CALL, + [CALL_PY_EXACT_ARGS_AWAITED] = CALL, [CALL_PY_GENERAL] = CALL, + [CALL_PY_GENERAL_AWAITED] = CALL, [CALL_STR_1] = CALL, [CALL_TUPLE_1] = CALL, [CALL_TYPE_1] = CALL, @@ -1929,6 +1977,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [DICT_MERGE] = DICT_MERGE, [DICT_UPDATE] = DICT_UPDATE, [END_ASYNC_FOR] = END_ASYNC_FOR, + [END_AWAITED_CALL] = END_AWAITED_CALL, [END_FOR] = END_FOR, [END_SEND] = END_SEND, [ENTER_EXECUTOR] = ENTER_EXECUTOR, @@ -2089,7 +2138,6 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ - case 117: \ case 118: \ case 119: \ case 120: \ @@ -2100,12 +2148,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 125: \ case 126: \ case 127: \ - case 219: \ - case 220: \ - case 221: \ - case 222: \ - case 223: \ - case 224: \ case 225: \ case 226: \ case 227: \ diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 7067b48ec22cb3..12cba6531fcc3f 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -94,6 +94,12 @@ extern "C" { #define RESUME_OPARG_LOCATION_MASK 0x7 #define RESUME_OPARG_DEPTH1_MASK 0x8 +/* Set in the oparg of CALL and CALL_KW when the result is awaited right away. + * The compiler never emits these calls with that many arguments (see + * _PY_STACK_USE_GUIDELINE), and awaited calls are never specialized, so only + * the generic forms need to mask it out. */ +#define CALL_AWAITED 0x80 + #define GET_ITER_YIELD_FROM 1 #define GET_ITER_YIELD_FROM_NO_CHECK 2 #define GET_ITER_YIELD_FROM_CORO_CHECK 3 diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ba6e0252024a6d..5a2a301c1f75d1 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -82,6 +82,7 @@ enum { _CHECK_IS_NOT_PY_CALLABLE_EX, _CHECK_IS_NOT_PY_CALLABLE_KW, _CHECK_IS_PY_CALLABLE_EX, + _CHECK_LAZY_CORO, _CHECK_MANAGED_OBJECT_HAS_VALUES, _CHECK_METHOD_VERSION, _CHECK_METHOD_VERSION_KW, @@ -95,6 +96,7 @@ enum { _CHECK_STACK_SPACE, _CHECK_STACK_SPACE_OPERAND, _CHECK_VALIDITY, + _CLEAR_CALL_AWAITED, _COLD_DYNAMIC_EXIT, _COLD_EXIT, _COMPARE_OP, @@ -116,6 +118,7 @@ enum { _DO_CALL_FUNCTION_EX, _DO_CALL_KW, _DYNAMIC_EXIT, + _END_AWAITED_CALL, _ERROR_POP_N, _EXPAND_METHOD, _EXPAND_METHOD_KW, @@ -128,6 +131,7 @@ enum { _GET_ITER, _GET_ITER_TRAD, _GUARD_3OS_ASYNC_GEN_ASEND, + _GUARD_AWAITED_CALL_NOT_LAZY, _GUARD_BINARY_OP_EXTEND, _GUARD_BINARY_OP_EXTEND_LHS, _GUARD_BINARY_OP_EXTEND_RHS, @@ -191,6 +195,7 @@ enum { _GUARD_NOT_EXHAUSTED_LIST, _GUARD_NOT_EXHAUSTED_RANGE, _GUARD_NOT_EXHAUSTED_TUPLE, + _GUARD_NOT_LAZY_CORO, _GUARD_THIRD_NULL, _GUARD_TOS_ANY_DICT, _GUARD_TOS_ANY_SET, @@ -287,12 +292,14 @@ enum { _MATCH_CLASS, _MAYBE_EXPAND_METHOD, _MAYBE_EXPAND_METHOD_KW, + _MAYBE_LAZY_CORO, _MONITOR_CALL, _MONITOR_CALL_KW, _MONITOR_JUMP_BACKWARD, _MONITOR_RESUME, _POP_JUMP_IF_FALSE, _POP_JUMP_IF_TRUE, + _POP_LAZY_OUTCOME, _POP_TOP_FLOAT, _POP_TOP_INT, _POP_TOP_NOP, @@ -328,6 +335,7 @@ enum { _SET_UPDATE, _SPILL_OR_RELOAD, _START_EXECUTOR, + _START_LAZY_CORO, _STORE_ATTR, _STORE_ATTR_INSTANCE_VALUE, _STORE_ATTR_SLOT, @@ -366,7 +374,7 @@ enum { _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, _YIELD_VALUE, }; -#define MAX_UOP_ID 654 +#define MAX_UOP_ID 662 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST @@ -590,6 +598,10 @@ enum { _CHECK_IS_PY_CALLABLE_EX_r13, _CHECK_IS_PY_CALLABLE_EX_r23, _CHECK_IS_PY_CALLABLE_EX_r33, + _CHECK_LAZY_CORO_r00, + _CHECK_LAZY_CORO_r11, + _CHECK_LAZY_CORO_r22, + _CHECK_LAZY_CORO_r33, _CHECK_MANAGED_OBJECT_HAS_VALUES_r01, _CHECK_MANAGED_OBJECT_HAS_VALUES_r11, _CHECK_MANAGED_OBJECT_HAS_VALUES_r22, @@ -621,6 +633,10 @@ enum { _CHECK_VALIDITY_r11, _CHECK_VALIDITY_r22, _CHECK_VALIDITY_r33, + _CLEAR_CALL_AWAITED_r00, + _CLEAR_CALL_AWAITED_r11, + _CLEAR_CALL_AWAITED_r22, + _CLEAR_CALL_AWAITED_r33, _COLD_DYNAMIC_EXIT_r00, _COLD_EXIT_r00, _COMPARE_OP_r21, @@ -665,6 +681,7 @@ enum { _DYNAMIC_EXIT_r10, _DYNAMIC_EXIT_r20, _DYNAMIC_EXIT_r30, + _END_AWAITED_CALL_r22, _END_FOR_r10, _END_SEND_r31, _ERROR_POP_N_r00, @@ -698,6 +715,10 @@ enum { _GUARD_3OS_ASYNC_GEN_ASEND_r13, _GUARD_3OS_ASYNC_GEN_ASEND_r23, _GUARD_3OS_ASYNC_GEN_ASEND_r33, + _GUARD_AWAITED_CALL_NOT_LAZY_r02, + _GUARD_AWAITED_CALL_NOT_LAZY_r12, + _GUARD_AWAITED_CALL_NOT_LAZY_r22, + _GUARD_AWAITED_CALL_NOT_LAZY_r33, _GUARD_BINARY_OP_EXTEND_r22, _GUARD_BINARY_OP_EXTEND_LHS_r02, _GUARD_BINARY_OP_EXTEND_LHS_r12, @@ -920,6 +941,10 @@ enum { _GUARD_NOT_EXHAUSTED_TUPLE_r12, _GUARD_NOT_EXHAUSTED_TUPLE_r22, _GUARD_NOT_EXHAUSTED_TUPLE_r33, + _GUARD_NOT_LAZY_CORO_r00, + _GUARD_NOT_LAZY_CORO_r11, + _GUARD_NOT_LAZY_CORO_r22, + _GUARD_NOT_LAZY_CORO_r33, _GUARD_THIRD_NULL_r03, _GUARD_THIRD_NULL_r13, _GUARD_THIRD_NULL_r23, @@ -1237,6 +1262,10 @@ enum { _MATCH_SEQUENCE_r23, _MAYBE_EXPAND_METHOD_r00, _MAYBE_EXPAND_METHOD_KW_r11, + _MAYBE_LAZY_CORO_r01, + _MAYBE_LAZY_CORO_r11, + _MAYBE_LAZY_CORO_r22, + _MAYBE_LAZY_CORO_r33, _MONITOR_CALL_r00, _MONITOR_CALL_KW_r11, _MONITOR_JUMP_BACKWARD_r00, @@ -1258,6 +1287,10 @@ enum { _POP_JUMP_IF_TRUE_r10, _POP_JUMP_IF_TRUE_r21, _POP_JUMP_IF_TRUE_r32, + _POP_LAZY_OUTCOME_r01, + _POP_LAZY_OUTCOME_r11, + _POP_LAZY_OUTCOME_r21, + _POP_LAZY_OUTCOME_r32, _POP_TOP_r10, _POP_TOP_FLOAT_r00, _POP_TOP_FLOAT_r10, @@ -1339,6 +1372,10 @@ enum { _SPILL_OR_RELOAD_r31, _SPILL_OR_RELOAD_r32, _START_EXECUTOR_r00, + _START_LAZY_CORO_r01, + _START_LAZY_CORO_r11, + _START_LAZY_CORO_r22, + _START_LAZY_CORO_r33, _STORE_ATTR_r20, _STORE_ATTR_INSTANCE_VALUE_r21, _STORE_ATTR_SLOT_r21, @@ -1444,7 +1481,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1655 +#define MAX_UOP_REGS_ID 1692 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 0cd82c53aa932a..d27231636878d2 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -168,12 +168,15 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_AWAITED_CALL_NOT_LAZY] = HAS_EXIT_FLAG, + [_POP_LAZY_OUTCOME] = 0, [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_GUARD_TOS_IS_NONE] = HAS_EXIT_FLAG, [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, [_SEND_VIRTUAL_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, [_GUARD_3OS_ASYNC_GEN_ASEND] = HAS_EXIT_FLAG, [_SEND_ASYNC_GEN_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_NOT_LAZY_CORO] = HAS_EXIT_FLAG, [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, @@ -1644,6 +1647,24 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_GUARD_AWAITED_CALL_NOT_LAZY] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_AWAITED_CALL_NOT_LAZY_r02 }, + { 2, 1, _GUARD_AWAITED_CALL_NOT_LAZY_r12 }, + { 2, 2, _GUARD_AWAITED_CALL_NOT_LAZY_r22 }, + { 3, 3, _GUARD_AWAITED_CALL_NOT_LAZY_r33 }, + }, + }, + [_POP_LAZY_OUTCOME] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _POP_LAZY_OUTCOME_r01 }, + { 1, 1, _POP_LAZY_OUTCOME_r11 }, + { 1, 2, _POP_LAZY_OUTCOME_r21 }, + { 2, 3, _POP_LAZY_OUTCOME_r32 }, + }, + }, [_SEND_GEN_FRAME] = { .best = { 3, 3, 3, 3 }, .entries = { @@ -1698,6 +1719,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _SEND_ASYNC_GEN_TIER_TWO_r33 }, }, }, + [_GUARD_NOT_LAZY_CORO] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_NOT_LAZY_CORO_r00 }, + { 1, 1, _GUARD_NOT_LAZY_CORO_r11 }, + { 2, 2, _GUARD_NOT_LAZY_CORO_r22 }, + { 3, 3, _GUARD_NOT_LAZY_CORO_r33 }, + }, + }, [_YIELD_VALUE] = { .best = { 1, 1, 1, 1 }, .entries = { @@ -4285,6 +4315,14 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GET_AITER_r11] = _GET_AITER, [_GET_ANEXT_r12] = _GET_ANEXT, [_GET_AWAITABLE_r11] = _GET_AWAITABLE, + [_GUARD_AWAITED_CALL_NOT_LAZY_r02] = _GUARD_AWAITED_CALL_NOT_LAZY, + [_GUARD_AWAITED_CALL_NOT_LAZY_r12] = _GUARD_AWAITED_CALL_NOT_LAZY, + [_GUARD_AWAITED_CALL_NOT_LAZY_r22] = _GUARD_AWAITED_CALL_NOT_LAZY, + [_GUARD_AWAITED_CALL_NOT_LAZY_r33] = _GUARD_AWAITED_CALL_NOT_LAZY, + [_POP_LAZY_OUTCOME_r01] = _POP_LAZY_OUTCOME, + [_POP_LAZY_OUTCOME_r11] = _POP_LAZY_OUTCOME, + [_POP_LAZY_OUTCOME_r21] = _POP_LAZY_OUTCOME, + [_POP_LAZY_OUTCOME_r32] = _POP_LAZY_OUTCOME, [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, [_GUARD_TOS_IS_NONE_r01] = _GUARD_TOS_IS_NONE, [_GUARD_TOS_IS_NONE_r11] = _GUARD_TOS_IS_NONE, @@ -4303,6 +4341,10 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_3OS_ASYNC_GEN_ASEND_r23] = _GUARD_3OS_ASYNC_GEN_ASEND, [_GUARD_3OS_ASYNC_GEN_ASEND_r33] = _GUARD_3OS_ASYNC_GEN_ASEND, [_SEND_ASYNC_GEN_TIER_TWO_r33] = _SEND_ASYNC_GEN_TIER_TWO, + [_GUARD_NOT_LAZY_CORO_r00] = _GUARD_NOT_LAZY_CORO, + [_GUARD_NOT_LAZY_CORO_r11] = _GUARD_NOT_LAZY_CORO, + [_GUARD_NOT_LAZY_CORO_r22] = _GUARD_NOT_LAZY_CORO, + [_GUARD_NOT_LAZY_CORO_r33] = _GUARD_NOT_LAZY_CORO, [_YIELD_VALUE_r11] = _YIELD_VALUE, [_POP_EXCEPT_r10] = _POP_EXCEPT, [_LOAD_COMMON_CONSTANT_r01] = _LOAD_COMMON_CONSTANT, @@ -5250,6 +5292,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_3OS_ASYNC_GEN_ASEND_r13] = "_GUARD_3OS_ASYNC_GEN_ASEND_r13", [_GUARD_3OS_ASYNC_GEN_ASEND_r23] = "_GUARD_3OS_ASYNC_GEN_ASEND_r23", [_GUARD_3OS_ASYNC_GEN_ASEND_r33] = "_GUARD_3OS_ASYNC_GEN_ASEND_r33", + [_GUARD_AWAITED_CALL_NOT_LAZY] = "_GUARD_AWAITED_CALL_NOT_LAZY", + [_GUARD_AWAITED_CALL_NOT_LAZY_r02] = "_GUARD_AWAITED_CALL_NOT_LAZY_r02", + [_GUARD_AWAITED_CALL_NOT_LAZY_r12] = "_GUARD_AWAITED_CALL_NOT_LAZY_r12", + [_GUARD_AWAITED_CALL_NOT_LAZY_r22] = "_GUARD_AWAITED_CALL_NOT_LAZY_r22", + [_GUARD_AWAITED_CALL_NOT_LAZY_r33] = "_GUARD_AWAITED_CALL_NOT_LAZY_r33", [_GUARD_BINARY_OP_EXTEND] = "_GUARD_BINARY_OP_EXTEND", [_GUARD_BINARY_OP_EXTEND_r22] = "_GUARD_BINARY_OP_EXTEND_r22", [_GUARD_BINARY_OP_EXTEND_LHS] = "_GUARD_BINARY_OP_EXTEND_LHS", @@ -5535,6 +5582,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_NOT_EXHAUSTED_TUPLE_r12] = "_GUARD_NOT_EXHAUSTED_TUPLE_r12", [_GUARD_NOT_EXHAUSTED_TUPLE_r22] = "_GUARD_NOT_EXHAUSTED_TUPLE_r22", [_GUARD_NOT_EXHAUSTED_TUPLE_r33] = "_GUARD_NOT_EXHAUSTED_TUPLE_r33", + [_GUARD_NOT_LAZY_CORO] = "_GUARD_NOT_LAZY_CORO", + [_GUARD_NOT_LAZY_CORO_r00] = "_GUARD_NOT_LAZY_CORO_r00", + [_GUARD_NOT_LAZY_CORO_r11] = "_GUARD_NOT_LAZY_CORO_r11", + [_GUARD_NOT_LAZY_CORO_r22] = "_GUARD_NOT_LAZY_CORO_r22", + [_GUARD_NOT_LAZY_CORO_r33] = "_GUARD_NOT_LAZY_CORO_r33", [_GUARD_THIRD_NULL] = "_GUARD_THIRD_NULL", [_GUARD_THIRD_NULL_r03] = "_GUARD_THIRD_NULL_r03", [_GUARD_THIRD_NULL_r13] = "_GUARD_THIRD_NULL_r13", @@ -5928,6 +5980,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_POP_EXCEPT_r10] = "_POP_EXCEPT_r10", [_POP_ITER] = "_POP_ITER", [_POP_ITER_r20] = "_POP_ITER_r20", + [_POP_LAZY_OUTCOME] = "_POP_LAZY_OUTCOME", + [_POP_LAZY_OUTCOME_r01] = "_POP_LAZY_OUTCOME_r01", + [_POP_LAZY_OUTCOME_r11] = "_POP_LAZY_OUTCOME_r11", + [_POP_LAZY_OUTCOME_r21] = "_POP_LAZY_OUTCOME_r21", + [_POP_LAZY_OUTCOME_r32] = "_POP_LAZY_OUTCOME_r32", [_POP_TOP] = "_POP_TOP", [_POP_TOP_r10] = "_POP_TOP_r10", [_POP_TOP_FLOAT] = "_POP_TOP_FLOAT", @@ -6464,6 +6521,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GET_AWAITABLE: return 1; + case _GUARD_AWAITED_CALL_NOT_LAZY: + return 0; + case _POP_LAZY_OUTCOME: + return 2; case _SEND_GEN_FRAME: return 1; case _GUARD_TOS_IS_NONE: @@ -6476,6 +6537,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _SEND_ASYNC_GEN_TIER_TWO: return 3; + case _GUARD_NOT_LAZY_CORO: + return 0; case _YIELD_VALUE: return 1; case _POP_EXCEPT: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 11342ae451b9f6..927400f9774170 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -14,53 +14,53 @@ extern "C" { #define BINARY_SLICE 1 #define BUILD_TEMPLATE 2 #define BINARY_OP_INPLACE_ADD_UNICODE 3 -#define CALL_FUNCTION_EX 4 -#define CHECK_EG_MATCH 5 -#define CHECK_EXC_MATCH 6 -#define CLEANUP_THROW 7 -#define DELETE_SUBSCR 8 -#define END_FOR 9 -#define END_SEND 10 -#define EXIT_INIT_CHECK 11 -#define FORMAT_SIMPLE 12 -#define FORMAT_WITH_SPEC 13 -#define GET_AITER 14 -#define GET_ANEXT 15 -#define GET_LEN 16 +#define CHECK_EG_MATCH 4 +#define CHECK_EXC_MATCH 5 +#define CLEANUP_THROW 6 +#define DELETE_SUBSCR 7 +#define END_FOR 8 +#define END_SEND 9 +#define EXIT_INIT_CHECK 10 +#define FORMAT_SIMPLE 11 +#define FORMAT_WITH_SPEC 12 +#define GET_AITER 13 +#define GET_ANEXT 14 +#define GET_LEN 15 +#define INTERPRETER_EXIT 16 #define RESERVED 17 -#define INTERPRETER_EXIT 18 -#define LOAD_BUILD_CLASS 19 -#define LOAD_LOCALS 20 -#define MAKE_FUNCTION 21 -#define MATCH_KEYS 22 -#define MATCH_MAPPING 23 -#define MATCH_SEQUENCE 24 -#define NOP 25 -#define NOT_TAKEN 26 -#define POP_EXCEPT 27 -#define POP_ITER 28 -#define POP_TOP 29 -#define PUSH_EXC_INFO 30 -#define PUSH_NULL 31 -#define RETURN_GENERATOR 32 -#define RETURN_VALUE 33 -#define SETUP_ANNOTATIONS 34 -#define STORE_SLICE 35 -#define STORE_SUBSCR 36 -#define TO_BOOL 37 -#define UNARY_INVERT 38 -#define UNARY_NEGATIVE 39 -#define UNARY_NOT 40 -#define WITH_EXCEPT_START 41 -#define BINARY_OP 42 -#define BUILD_INTERPOLATION 43 -#define BUILD_LIST 44 -#define BUILD_MAP 45 -#define BUILD_SET 46 -#define BUILD_SLICE 47 -#define BUILD_STRING 48 -#define BUILD_TUPLE 49 -#define CALL 50 +#define LOAD_BUILD_CLASS 18 +#define LOAD_LOCALS 19 +#define MAKE_FUNCTION 20 +#define MATCH_KEYS 21 +#define MATCH_MAPPING 22 +#define MATCH_SEQUENCE 23 +#define NOP 24 +#define NOT_TAKEN 25 +#define POP_EXCEPT 26 +#define POP_ITER 27 +#define POP_TOP 28 +#define PUSH_EXC_INFO 29 +#define PUSH_NULL 30 +#define RETURN_GENERATOR 31 +#define RETURN_VALUE 32 +#define SETUP_ANNOTATIONS 33 +#define STORE_SLICE 34 +#define STORE_SUBSCR 35 +#define TO_BOOL 36 +#define UNARY_INVERT 37 +#define UNARY_NEGATIVE 38 +#define UNARY_NOT 39 +#define WITH_EXCEPT_START 40 +#define BINARY_OP 41 +#define BUILD_INTERPOLATION 42 +#define BUILD_LIST 43 +#define BUILD_MAP 44 +#define BUILD_SET 45 +#define BUILD_SLICE 46 +#define BUILD_STRING 47 +#define BUILD_TUPLE 48 +#define CALL 49 +#define CALL_FUNCTION_EX 50 #define CALL_INTRINSIC_1 51 #define CALL_INTRINSIC_2 52 #define CALL_KW 53 @@ -74,59 +74,60 @@ extern "C" { #define DICT_MERGE 61 #define DICT_UPDATE 62 #define END_ASYNC_FOR 63 -#define EXTENDED_ARG 64 -#define FOR_ITER 65 -#define GET_AWAITABLE 66 -#define GET_ITER 67 -#define IMPORT_FROM 68 -#define IMPORT_NAME 69 -#define IS_OP 70 -#define JUMP_BACKWARD 71 -#define JUMP_BACKWARD_NO_INTERRUPT 72 -#define JUMP_FORWARD 73 -#define LIST_APPEND 74 -#define LIST_EXTEND 75 -#define LOAD_ATTR 76 -#define LOAD_COMMON_CONSTANT 77 -#define LOAD_CONST 78 -#define LOAD_DEREF 79 -#define LOAD_FAST 80 -#define LOAD_FAST_AND_CLEAR 81 -#define LOAD_FAST_BORROW 82 -#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 83 -#define LOAD_FAST_CHECK 84 -#define LOAD_FAST_LOAD_FAST 85 -#define LOAD_FROM_DICT_OR_DEREF 86 -#define LOAD_FROM_DICT_OR_GLOBALS 87 -#define LOAD_GLOBAL 88 -#define LOAD_NAME 89 -#define LOAD_SMALL_INT 90 -#define LOAD_SPECIAL 91 -#define LOAD_SUPER_ATTR 92 -#define MAKE_CELL 93 -#define MAP_ADD 94 -#define MATCH_CLASS 95 -#define POP_JUMP_IF_FALSE 96 -#define POP_JUMP_IF_NONE 97 -#define POP_JUMP_IF_NOT_NONE 98 -#define POP_JUMP_IF_TRUE 99 -#define RAISE_VARARGS 100 -#define RERAISE 101 -#define SEND 102 -#define SET_ADD 103 -#define SET_FUNCTION_ATTRIBUTE 104 -#define SET_UPDATE 105 -#define STORE_ATTR 106 -#define STORE_DEREF 107 -#define STORE_FAST 108 -#define STORE_FAST_LOAD_FAST 109 -#define STORE_FAST_STORE_FAST 110 -#define STORE_GLOBAL 111 -#define STORE_NAME 112 -#define SWAP 113 -#define UNPACK_EX 114 -#define UNPACK_SEQUENCE 115 -#define YIELD_VALUE 116 +#define END_AWAITED_CALL 64 +#define EXTENDED_ARG 65 +#define FOR_ITER 66 +#define GET_AWAITABLE 67 +#define GET_ITER 68 +#define IMPORT_FROM 69 +#define IMPORT_NAME 70 +#define IS_OP 71 +#define JUMP_BACKWARD 72 +#define JUMP_BACKWARD_NO_INTERRUPT 73 +#define JUMP_FORWARD 74 +#define LIST_APPEND 75 +#define LIST_EXTEND 76 +#define LOAD_ATTR 77 +#define LOAD_COMMON_CONSTANT 78 +#define LOAD_CONST 79 +#define LOAD_DEREF 80 +#define LOAD_FAST 81 +#define LOAD_FAST_AND_CLEAR 82 +#define LOAD_FAST_BORROW 83 +#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 84 +#define LOAD_FAST_CHECK 85 +#define LOAD_FAST_LOAD_FAST 86 +#define LOAD_FROM_DICT_OR_DEREF 87 +#define LOAD_FROM_DICT_OR_GLOBALS 88 +#define LOAD_GLOBAL 89 +#define LOAD_NAME 90 +#define LOAD_SMALL_INT 91 +#define LOAD_SPECIAL 92 +#define LOAD_SUPER_ATTR 93 +#define MAKE_CELL 94 +#define MAP_ADD 95 +#define MATCH_CLASS 96 +#define POP_JUMP_IF_FALSE 97 +#define POP_JUMP_IF_NONE 98 +#define POP_JUMP_IF_NOT_NONE 99 +#define POP_JUMP_IF_TRUE 100 +#define RAISE_VARARGS 101 +#define RERAISE 102 +#define SEND 103 +#define SET_ADD 104 +#define SET_FUNCTION_ATTRIBUTE 105 +#define SET_UPDATE 106 +#define STORE_ATTR 107 +#define STORE_DEREF 108 +#define STORE_FAST 109 +#define STORE_FAST_LOAD_FAST 110 +#define STORE_FAST_STORE_FAST 111 +#define STORE_GLOBAL 112 +#define STORE_NAME 113 +#define SWAP 114 +#define UNPACK_EX 115 +#define UNPACK_SEQUENCE 116 +#define YIELD_VALUE 117 #define RESUME 128 #define BINARY_OP_ADD_FLOAT 129 #define BINARY_OP_ADD_INT 130 @@ -145,79 +146,85 @@ extern "C" { #define BINARY_OP_SUBTRACT_INT 143 #define CALL_ALLOC_AND_ENTER_INIT 144 #define CALL_BOUND_METHOD_EXACT_ARGS 145 -#define CALL_BOUND_METHOD_GENERAL 146 -#define CALL_BUILTIN_CLASS 147 -#define CALL_BUILTIN_FAST 148 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 149 -#define CALL_BUILTIN_O 150 -#define CALL_EX_NON_PY_GENERAL 151 -#define CALL_EX_PY 152 -#define CALL_ISINSTANCE 153 -#define CALL_KW_BOUND_METHOD 154 -#define CALL_KW_NON_PY 155 -#define CALL_KW_PY 156 -#define CALL_LEN 157 -#define CALL_LIST_APPEND 158 -#define CALL_METHOD_DESCRIPTOR_FAST 159 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 160 -#define CALL_METHOD_DESCRIPTOR_NOARGS 161 -#define CALL_METHOD_DESCRIPTOR_O 162 -#define CALL_NON_PY_GENERAL 163 -#define CALL_PY_EXACT_ARGS 164 -#define CALL_PY_GENERAL 165 -#define CALL_STR_1 166 -#define CALL_TUPLE_1 167 -#define CALL_TYPE_1 168 -#define COMPARE_OP_FLOAT 169 -#define COMPARE_OP_INT 170 -#define COMPARE_OP_STR 171 -#define CONTAINS_OP_DICT 172 -#define CONTAINS_OP_SET 173 -#define FOR_ITER_GEN 174 -#define FOR_ITER_LIST 175 -#define FOR_ITER_RANGE 176 -#define FOR_ITER_TUPLE 177 -#define FOR_ITER_VIRTUAL 178 -#define GET_ITER_SELF 179 -#define GET_ITER_VIRTUAL 180 -#define JUMP_BACKWARD_JIT 181 -#define JUMP_BACKWARD_NO_JIT 182 -#define LOAD_ATTR_CLASS 183 -#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 184 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 185 -#define LOAD_ATTR_INSTANCE_VALUE 186 -#define LOAD_ATTR_METHOD_LAZY_DICT 187 -#define LOAD_ATTR_METHOD_NO_DICT 188 -#define LOAD_ATTR_METHOD_WITH_VALUES 189 -#define LOAD_ATTR_MODULE 190 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 191 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 192 -#define LOAD_ATTR_PROPERTY 193 -#define LOAD_ATTR_SLOT 194 -#define LOAD_ATTR_WITH_HINT 195 -#define LOAD_GLOBAL_BUILTIN 196 -#define LOAD_GLOBAL_MODULE 197 -#define LOAD_SUPER_ATTR_ATTR 198 -#define LOAD_SUPER_ATTR_METHOD 199 -#define RESUME_CHECK 200 -#define RESUME_CHECK_JIT 201 -#define SEND_ASYNC_GEN 202 -#define SEND_GEN 203 -#define SEND_VIRTUAL 204 -#define STORE_ATTR_INSTANCE_VALUE 205 -#define STORE_ATTR_SLOT 206 -#define STORE_ATTR_WITH_HINT 207 -#define STORE_SUBSCR_DICT 208 -#define STORE_SUBSCR_LIST_INT 209 -#define TO_BOOL_ALWAYS_TRUE 210 -#define TO_BOOL_BOOL 211 -#define TO_BOOL_INT 212 -#define TO_BOOL_LIST 213 -#define TO_BOOL_NONE 214 -#define TO_BOOL_STR 215 -#define UNPACK_SEQUENCE_LIST 216 -#define UNPACK_SEQUENCE_TUPLE 217 -#define UNPACK_SEQUENCE_TWO_TUPLE 218 +#define CALL_BOUND_METHOD_EXACT_ARGS_AWAITED 146 +#define CALL_BOUND_METHOD_GENERAL 147 +#define CALL_BOUND_METHOD_GENERAL_AWAITED 148 +#define CALL_BUILTIN_CLASS 149 +#define CALL_BUILTIN_FAST 150 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 151 +#define CALL_BUILTIN_O 152 +#define CALL_EX_NON_PY_GENERAL 153 +#define CALL_EX_PY 154 +#define CALL_ISINSTANCE 155 +#define CALL_KW_BOUND_METHOD 156 +#define CALL_KW_BOUND_METHOD_AWAITED 157 +#define CALL_KW_NON_PY 158 +#define CALL_KW_PY 159 +#define CALL_KW_PY_AWAITED 160 +#define CALL_LEN 161 +#define CALL_LIST_APPEND 162 +#define CALL_METHOD_DESCRIPTOR_FAST 163 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 164 +#define CALL_METHOD_DESCRIPTOR_NOARGS 165 +#define CALL_METHOD_DESCRIPTOR_O 166 +#define CALL_NON_PY_GENERAL 167 +#define CALL_PY_EXACT_ARGS 168 +#define CALL_PY_EXACT_ARGS_AWAITED 169 +#define CALL_PY_GENERAL 170 +#define CALL_PY_GENERAL_AWAITED 171 +#define CALL_STR_1 172 +#define CALL_TUPLE_1 173 +#define CALL_TYPE_1 174 +#define COMPARE_OP_FLOAT 175 +#define COMPARE_OP_INT 176 +#define COMPARE_OP_STR 177 +#define CONTAINS_OP_DICT 178 +#define CONTAINS_OP_SET 179 +#define FOR_ITER_GEN 180 +#define FOR_ITER_LIST 181 +#define FOR_ITER_RANGE 182 +#define FOR_ITER_TUPLE 183 +#define FOR_ITER_VIRTUAL 184 +#define GET_ITER_SELF 185 +#define GET_ITER_VIRTUAL 186 +#define JUMP_BACKWARD_JIT 187 +#define JUMP_BACKWARD_NO_JIT 188 +#define LOAD_ATTR_CLASS 189 +#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 190 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 191 +#define LOAD_ATTR_INSTANCE_VALUE 192 +#define LOAD_ATTR_METHOD_LAZY_DICT 193 +#define LOAD_ATTR_METHOD_NO_DICT 194 +#define LOAD_ATTR_METHOD_WITH_VALUES 195 +#define LOAD_ATTR_MODULE 196 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 197 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 198 +#define LOAD_ATTR_PROPERTY 199 +#define LOAD_ATTR_SLOT 200 +#define LOAD_ATTR_WITH_HINT 201 +#define LOAD_GLOBAL_BUILTIN 202 +#define LOAD_GLOBAL_MODULE 203 +#define LOAD_SUPER_ATTR_ATTR 204 +#define LOAD_SUPER_ATTR_METHOD 205 +#define RESUME_CHECK 206 +#define RESUME_CHECK_JIT 207 +#define SEND_ASYNC_GEN 208 +#define SEND_GEN 209 +#define SEND_VIRTUAL 210 +#define STORE_ATTR_INSTANCE_VALUE 211 +#define STORE_ATTR_SLOT 212 +#define STORE_ATTR_WITH_HINT 213 +#define STORE_SUBSCR_DICT 214 +#define STORE_SUBSCR_LIST_INT 215 +#define TO_BOOL_ALWAYS_TRUE 216 +#define TO_BOOL_BOOL 217 +#define TO_BOOL_INT 218 +#define TO_BOOL_LIST 219 +#define TO_BOOL_NONE 220 +#define TO_BOOL_STR 221 +#define UNPACK_SEQUENCE_LIST 222 +#define UNPACK_SEQUENCE_TUPLE 223 +#define UNPACK_SEQUENCE_TWO_TUPLE 224 #define INSTRUMENTED_END_FOR 233 #define INSTRUMENTED_POP_ITER 234 #define INSTRUMENTED_END_SEND 235 @@ -253,7 +260,7 @@ extern "C" { #define SETUP_WITH 265 #define STORE_FAST_MAYBE_NULL 266 -#define HAVE_ARGUMENT 41 +#define HAVE_ARGUMENT 40 #define MIN_SPECIALIZED_OPCODE 129 #define MIN_INSTRUMENTED_OPCODE 233 diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index fa652cd231cfa9..ac3036a477969f 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -45,7 +45,37 @@ The generator's frame is initialized as a copy of the frame in which `RETURN_GENERATOR` is executing, but its `owner` field is overwritten to indicate that it is owned by a generator. Finally, `RETURN_GENERATOR` pushes the new generator object to the stack and returns to the caller of the generator function (at -which time its frame is destroyed). When the generator is next resumed by +which time its frame is destroyed). + +There is one exception: a coroutine function called by an awaited call +(`await f()`, compiled as `CALL`, `CALL_KW` or `CALL_FUNCTION_EX` with the low +bit of the oparg set) gets the `lazy_coro` flag on its frame (see +`_MAYBE_LAZY_CORO`). `RETURN_GENERATOR` then keeps running the body in that +frame, with its own `_PyErr_StackItem` pushed on the data stack right after the +frame, so that a body that returns or raises without suspending never creates a +coroutine object. + +The compiler pushes an extra `NULL`, the `lazy_outcome`, below the callable of +an awaited call and follows the call with `END_AWAITED_CALL`, whose oparg jumps +to the `YIELD_VALUE` of the `SEND` loop for the await. A lazily run body reports +in `lazy_outcome` how it ended: + +- It returned: `clear_thread_frame()` stores a tagged int, and + `END_AWAITED_CALL` skips the `SEND` loop, jumping past the `END_SEND` that + the `SEND` right before `YIELD_VALUE` jumps to. +- It suspended: `YIELD_VALUE` leaves the frame and its `_PyErr_StackItem` on the + data stack and stores a pointer to the frame. `END_AWAITED_CALL` creates the + coroutine object, moves the frame and the exception state into it (see + `_PyEval_MaterializeLazyCoro()`) and jumps to `YIELD_VALUE` with the + coroutine, `NULL` and the yielded value on the stack, as `SEND` would have + left them. +- It raised: the exception is raised from the caller's `END_AWAITED_CALL`, + which has the location of the `await`. + +Otherwise `lazy_outcome` stays `NULL`, and `END_AWAITED_CALL` pops it and goes +on to `GET_AWAITABLE`. + +When the generator is next resumed by [`gen_send_ex2()`](../Objects/genobject.c), `_PyEval_EvalFrame()` is called to continue executing the generator function, in the frame that is embedded in the generator object. diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index df92eae151d248..3d8f58b8c8a00b 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -120,11 +120,17 @@ "CALL_PY_GENERAL", "CALL_BOUND_METHOD_GENERAL", "CALL_NON_PY_GENERAL", + "CALL_PY_EXACT_ARGS_AWAITED", + "CALL_BOUND_METHOD_EXACT_ARGS_AWAITED", + "CALL_PY_GENERAL_AWAITED", + "CALL_BOUND_METHOD_GENERAL_AWAITED", ), CALL_KW=( "CALL_KW_BOUND_METHOD", "CALL_KW_PY", "CALL_KW_NON_PY", + "CALL_KW_PY_AWAITED", + "CALL_KW_BOUND_METHOD_AWAITED", ), CALL_FUNCTION_EX=( "CALL_EX_PY", @@ -151,79 +157,85 @@ BINARY_OP_SUBTRACT_INT=143, CALL_ALLOC_AND_ENTER_INIT=144, CALL_BOUND_METHOD_EXACT_ARGS=145, - CALL_BOUND_METHOD_GENERAL=146, - CALL_BUILTIN_CLASS=147, - CALL_BUILTIN_FAST=148, - CALL_BUILTIN_FAST_WITH_KEYWORDS=149, - CALL_BUILTIN_O=150, - CALL_EX_NON_PY_GENERAL=151, - CALL_EX_PY=152, - CALL_ISINSTANCE=153, - CALL_KW_BOUND_METHOD=154, - CALL_KW_NON_PY=155, - CALL_KW_PY=156, - CALL_LEN=157, - CALL_LIST_APPEND=158, - CALL_METHOD_DESCRIPTOR_FAST=159, - CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS=160, - CALL_METHOD_DESCRIPTOR_NOARGS=161, - CALL_METHOD_DESCRIPTOR_O=162, - CALL_NON_PY_GENERAL=163, - CALL_PY_EXACT_ARGS=164, - CALL_PY_GENERAL=165, - CALL_STR_1=166, - CALL_TUPLE_1=167, - CALL_TYPE_1=168, - COMPARE_OP_FLOAT=169, - COMPARE_OP_INT=170, - COMPARE_OP_STR=171, - CONTAINS_OP_DICT=172, - CONTAINS_OP_SET=173, - FOR_ITER_GEN=174, - FOR_ITER_LIST=175, - FOR_ITER_RANGE=176, - FOR_ITER_TUPLE=177, - FOR_ITER_VIRTUAL=178, - GET_ITER_SELF=179, - GET_ITER_VIRTUAL=180, - JUMP_BACKWARD_JIT=181, - JUMP_BACKWARD_NO_JIT=182, - LOAD_ATTR_CLASS=183, - LOAD_ATTR_CLASS_WITH_METACLASS_CHECK=184, - LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN=185, - LOAD_ATTR_INSTANCE_VALUE=186, - LOAD_ATTR_METHOD_LAZY_DICT=187, - LOAD_ATTR_METHOD_NO_DICT=188, - LOAD_ATTR_METHOD_WITH_VALUES=189, - LOAD_ATTR_MODULE=190, - LOAD_ATTR_NONDESCRIPTOR_NO_DICT=191, - LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES=192, - LOAD_ATTR_PROPERTY=193, - LOAD_ATTR_SLOT=194, - LOAD_ATTR_WITH_HINT=195, - LOAD_GLOBAL_BUILTIN=196, - LOAD_GLOBAL_MODULE=197, - LOAD_SUPER_ATTR_ATTR=198, - LOAD_SUPER_ATTR_METHOD=199, - RESUME_CHECK=200, - RESUME_CHECK_JIT=201, - SEND_ASYNC_GEN=202, - SEND_GEN=203, - SEND_VIRTUAL=204, - STORE_ATTR_INSTANCE_VALUE=205, - STORE_ATTR_SLOT=206, - STORE_ATTR_WITH_HINT=207, - STORE_SUBSCR_DICT=208, - STORE_SUBSCR_LIST_INT=209, - TO_BOOL_ALWAYS_TRUE=210, - TO_BOOL_BOOL=211, - TO_BOOL_INT=212, - TO_BOOL_LIST=213, - TO_BOOL_NONE=214, - TO_BOOL_STR=215, - UNPACK_SEQUENCE_LIST=216, - UNPACK_SEQUENCE_TUPLE=217, - UNPACK_SEQUENCE_TWO_TUPLE=218, + CALL_BOUND_METHOD_EXACT_ARGS_AWAITED=146, + CALL_BOUND_METHOD_GENERAL=147, + CALL_BOUND_METHOD_GENERAL_AWAITED=148, + CALL_BUILTIN_CLASS=149, + CALL_BUILTIN_FAST=150, + CALL_BUILTIN_FAST_WITH_KEYWORDS=151, + CALL_BUILTIN_O=152, + CALL_EX_NON_PY_GENERAL=153, + CALL_EX_PY=154, + CALL_ISINSTANCE=155, + CALL_KW_BOUND_METHOD=156, + CALL_KW_BOUND_METHOD_AWAITED=157, + CALL_KW_NON_PY=158, + CALL_KW_PY=159, + CALL_KW_PY_AWAITED=160, + CALL_LEN=161, + CALL_LIST_APPEND=162, + CALL_METHOD_DESCRIPTOR_FAST=163, + CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS=164, + CALL_METHOD_DESCRIPTOR_NOARGS=165, + CALL_METHOD_DESCRIPTOR_O=166, + CALL_NON_PY_GENERAL=167, + CALL_PY_EXACT_ARGS=168, + CALL_PY_EXACT_ARGS_AWAITED=169, + CALL_PY_GENERAL=170, + CALL_PY_GENERAL_AWAITED=171, + CALL_STR_1=172, + CALL_TUPLE_1=173, + CALL_TYPE_1=174, + COMPARE_OP_FLOAT=175, + COMPARE_OP_INT=176, + COMPARE_OP_STR=177, + CONTAINS_OP_DICT=178, + CONTAINS_OP_SET=179, + FOR_ITER_GEN=180, + FOR_ITER_LIST=181, + FOR_ITER_RANGE=182, + FOR_ITER_TUPLE=183, + FOR_ITER_VIRTUAL=184, + GET_ITER_SELF=185, + GET_ITER_VIRTUAL=186, + JUMP_BACKWARD_JIT=187, + JUMP_BACKWARD_NO_JIT=188, + LOAD_ATTR_CLASS=189, + LOAD_ATTR_CLASS_WITH_METACLASS_CHECK=190, + LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN=191, + LOAD_ATTR_INSTANCE_VALUE=192, + LOAD_ATTR_METHOD_LAZY_DICT=193, + LOAD_ATTR_METHOD_NO_DICT=194, + LOAD_ATTR_METHOD_WITH_VALUES=195, + LOAD_ATTR_MODULE=196, + LOAD_ATTR_NONDESCRIPTOR_NO_DICT=197, + LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES=198, + LOAD_ATTR_PROPERTY=199, + LOAD_ATTR_SLOT=200, + LOAD_ATTR_WITH_HINT=201, + LOAD_GLOBAL_BUILTIN=202, + LOAD_GLOBAL_MODULE=203, + LOAD_SUPER_ATTR_ATTR=204, + LOAD_SUPER_ATTR_METHOD=205, + RESUME_CHECK=206, + RESUME_CHECK_JIT=207, + SEND_ASYNC_GEN=208, + SEND_GEN=209, + SEND_VIRTUAL=210, + STORE_ATTR_INSTANCE_VALUE=211, + STORE_ATTR_SLOT=212, + STORE_ATTR_WITH_HINT=213, + STORE_SUBSCR_DICT=214, + STORE_SUBSCR_LIST_INT=215, + TO_BOOL_ALWAYS_TRUE=216, + TO_BOOL_BOOL=217, + TO_BOOL_INT=218, + TO_BOOL_LIST=219, + TO_BOOL_NONE=220, + TO_BOOL_STR=221, + UNPACK_SEQUENCE_LIST=222, + UNPACK_SEQUENCE_TUPLE=223, + UNPACK_SEQUENCE_TWO_TUPLE=224, ) opmap = frozendict( @@ -235,52 +247,52 @@ TRACE_RECORD=255, BINARY_SLICE=1, BUILD_TEMPLATE=2, - CALL_FUNCTION_EX=4, - CHECK_EG_MATCH=5, - CHECK_EXC_MATCH=6, - CLEANUP_THROW=7, - DELETE_SUBSCR=8, - END_FOR=9, - END_SEND=10, - EXIT_INIT_CHECK=11, - FORMAT_SIMPLE=12, - FORMAT_WITH_SPEC=13, - GET_AITER=14, - GET_ANEXT=15, - GET_LEN=16, - INTERPRETER_EXIT=18, - LOAD_BUILD_CLASS=19, - LOAD_LOCALS=20, - MAKE_FUNCTION=21, - MATCH_KEYS=22, - MATCH_MAPPING=23, - MATCH_SEQUENCE=24, - NOP=25, - NOT_TAKEN=26, - POP_EXCEPT=27, - POP_ITER=28, - POP_TOP=29, - PUSH_EXC_INFO=30, - PUSH_NULL=31, - RETURN_GENERATOR=32, - RETURN_VALUE=33, - SETUP_ANNOTATIONS=34, - STORE_SLICE=35, - STORE_SUBSCR=36, - TO_BOOL=37, - UNARY_INVERT=38, - UNARY_NEGATIVE=39, - UNARY_NOT=40, - WITH_EXCEPT_START=41, - BINARY_OP=42, - BUILD_INTERPOLATION=43, - BUILD_LIST=44, - BUILD_MAP=45, - BUILD_SET=46, - BUILD_SLICE=47, - BUILD_STRING=48, - BUILD_TUPLE=49, - CALL=50, + CHECK_EG_MATCH=4, + CHECK_EXC_MATCH=5, + CLEANUP_THROW=6, + DELETE_SUBSCR=7, + END_FOR=8, + END_SEND=9, + EXIT_INIT_CHECK=10, + FORMAT_SIMPLE=11, + FORMAT_WITH_SPEC=12, + GET_AITER=13, + GET_ANEXT=14, + GET_LEN=15, + INTERPRETER_EXIT=16, + LOAD_BUILD_CLASS=18, + LOAD_LOCALS=19, + MAKE_FUNCTION=20, + MATCH_KEYS=21, + MATCH_MAPPING=22, + MATCH_SEQUENCE=23, + NOP=24, + NOT_TAKEN=25, + POP_EXCEPT=26, + POP_ITER=27, + POP_TOP=28, + PUSH_EXC_INFO=29, + PUSH_NULL=30, + RETURN_GENERATOR=31, + RETURN_VALUE=32, + SETUP_ANNOTATIONS=33, + STORE_SLICE=34, + STORE_SUBSCR=35, + TO_BOOL=36, + UNARY_INVERT=37, + UNARY_NEGATIVE=38, + UNARY_NOT=39, + WITH_EXCEPT_START=40, + BINARY_OP=41, + BUILD_INTERPOLATION=42, + BUILD_LIST=43, + BUILD_MAP=44, + BUILD_SET=45, + BUILD_SLICE=46, + BUILD_STRING=47, + BUILD_TUPLE=48, + CALL=49, + CALL_FUNCTION_EX=50, CALL_INTRINSIC_1=51, CALL_INTRINSIC_2=52, CALL_KW=53, @@ -294,59 +306,60 @@ DICT_MERGE=61, DICT_UPDATE=62, END_ASYNC_FOR=63, - EXTENDED_ARG=64, - FOR_ITER=65, - GET_AWAITABLE=66, - GET_ITER=67, - IMPORT_FROM=68, - IMPORT_NAME=69, - IS_OP=70, - JUMP_BACKWARD=71, - JUMP_BACKWARD_NO_INTERRUPT=72, - JUMP_FORWARD=73, - LIST_APPEND=74, - LIST_EXTEND=75, - LOAD_ATTR=76, - LOAD_COMMON_CONSTANT=77, - LOAD_CONST=78, - LOAD_DEREF=79, - LOAD_FAST=80, - LOAD_FAST_AND_CLEAR=81, - LOAD_FAST_BORROW=82, - LOAD_FAST_BORROW_LOAD_FAST_BORROW=83, - LOAD_FAST_CHECK=84, - LOAD_FAST_LOAD_FAST=85, - LOAD_FROM_DICT_OR_DEREF=86, - LOAD_FROM_DICT_OR_GLOBALS=87, - LOAD_GLOBAL=88, - LOAD_NAME=89, - LOAD_SMALL_INT=90, - LOAD_SPECIAL=91, - LOAD_SUPER_ATTR=92, - MAKE_CELL=93, - MAP_ADD=94, - MATCH_CLASS=95, - POP_JUMP_IF_FALSE=96, - POP_JUMP_IF_NONE=97, - POP_JUMP_IF_NOT_NONE=98, - POP_JUMP_IF_TRUE=99, - RAISE_VARARGS=100, - RERAISE=101, - SEND=102, - SET_ADD=103, - SET_FUNCTION_ATTRIBUTE=104, - SET_UPDATE=105, - STORE_ATTR=106, - STORE_DEREF=107, - STORE_FAST=108, - STORE_FAST_LOAD_FAST=109, - STORE_FAST_STORE_FAST=110, - STORE_GLOBAL=111, - STORE_NAME=112, - SWAP=113, - UNPACK_EX=114, - UNPACK_SEQUENCE=115, - YIELD_VALUE=116, + END_AWAITED_CALL=64, + EXTENDED_ARG=65, + FOR_ITER=66, + GET_AWAITABLE=67, + GET_ITER=68, + IMPORT_FROM=69, + IMPORT_NAME=70, + IS_OP=71, + JUMP_BACKWARD=72, + JUMP_BACKWARD_NO_INTERRUPT=73, + JUMP_FORWARD=74, + LIST_APPEND=75, + LIST_EXTEND=76, + LOAD_ATTR=77, + LOAD_COMMON_CONSTANT=78, + LOAD_CONST=79, + LOAD_DEREF=80, + LOAD_FAST=81, + LOAD_FAST_AND_CLEAR=82, + LOAD_FAST_BORROW=83, + LOAD_FAST_BORROW_LOAD_FAST_BORROW=84, + LOAD_FAST_CHECK=85, + LOAD_FAST_LOAD_FAST=86, + LOAD_FROM_DICT_OR_DEREF=87, + LOAD_FROM_DICT_OR_GLOBALS=88, + LOAD_GLOBAL=89, + LOAD_NAME=90, + LOAD_SMALL_INT=91, + LOAD_SPECIAL=92, + LOAD_SUPER_ATTR=93, + MAKE_CELL=94, + MAP_ADD=95, + MATCH_CLASS=96, + POP_JUMP_IF_FALSE=97, + POP_JUMP_IF_NONE=98, + POP_JUMP_IF_NOT_NONE=99, + POP_JUMP_IF_TRUE=100, + RAISE_VARARGS=101, + RERAISE=102, + SEND=103, + SET_ADD=104, + SET_FUNCTION_ATTRIBUTE=105, + SET_UPDATE=106, + STORE_ATTR=107, + STORE_DEREF=108, + STORE_FAST=109, + STORE_FAST_LOAD_FAST=110, + STORE_FAST_STORE_FAST=111, + STORE_GLOBAL=112, + STORE_NAME=113, + SWAP=114, + UNPACK_EX=115, + UNPACK_SEQUENCE=116, + YIELD_VALUE=117, INSTRUMENTED_END_FOR=233, INSTRUMENTED_POP_ITER=234, INSTRUMENTED_END_SEND=235, @@ -380,5 +393,5 @@ STORE_FAST_MAYBE_NULL=266, ) -HAVE_ARGUMENT = 41 +HAVE_ARGUMENT = 40 MIN_INSTRUMENTED_OPCODE = 233 diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index d76d2f6383343d..d7714c9c681a19 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -1,6 +1,7 @@ """Introspection utils for tasks call graphs.""" import dataclasses +import inspect import io import operator import sys @@ -38,6 +39,26 @@ class FutureCallGraph: awaited_by: tuple["FutureCallGraph", ...] +# A coroutine that is awaited right away runs without a coroutine +# object until it first suspends, so ``f.f_generator`` may be None +# for a coroutine frame; classify frames by their code instead. +_CO_ASYNC_FLAGS = inspect.CO_COROUTINE | inspect.CO_ASYNC_GENERATOR +_CO_GEN_FLAGS = _CO_ASYNC_FLAGS | inspect.CO_GENERATOR +_FRAME_TAGS = { + inspect.CO_COROUTINE: 'async ', + inspect.CO_ASYNC_GENERATOR: 'async generator ', + inspect.CO_GENERATOR: 'generator ', +} + + +def _is_async_frame(f): + return bool(f.f_code.co_flags & _CO_ASYNC_FLAGS) + + +def _is_sync_frame(f): + return not (f.f_code.co_flags & _CO_GEN_FLAGS) + + def _build_graph_for_future( future: futures.Future, *, @@ -166,12 +187,10 @@ def capture_call_graph( try: while f is not None: # gh-156988: sync gen should not clear the call chain - is_async = isinstance( - f.f_generator, (types.CoroutineType, types.AsyncGeneratorType)) call_stack.append(FrameCallGraphEntry(f)) - if is_async: - if f.f_back is not None and f.f_back.f_generator is None: + if _is_async_frame(f): + if f.f_back is not None and _is_sync_frame(f.f_back): # We've reached the bottom of the coroutine stack, which # must be the Task that runs it. break @@ -228,36 +247,12 @@ def add_line(line: str) -> None: ) for ste in st.call_stack: f = ste.frame - - if f.f_generator is None: - f = ste.frame - add_line( - f' | File {f.f_code.co_filename!r},' - f' line {f.f_lineno}, in' - f' {f.f_code.co_qualname}()' - ) - else: - c = f.f_generator - - try: - f = c.cr_frame - code = c.cr_code - tag = 'async' - except AttributeError: - try: - f = c.ag_frame - code = c.ag_code - tag = 'async generator' - except AttributeError: - f = c.gi_frame - code = c.gi_code - tag = 'generator' - - add_line( - f' | File {f.f_code.co_filename!r},' - f' line {f.f_lineno}, in' - f' {tag} {code.co_qualname}()' - ) + tag = _FRAME_TAGS.get(f.f_code.co_flags & _CO_GEN_FLAGS, '') + add_line( + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {tag}{f.f_code.co_qualname}()' + ) if st.awaited_by: add_line( diff --git a/Lib/dis.py b/Lib/dis.py index 318729091fa098..58ef45104ac81e 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -44,6 +44,10 @@ SEND = opmap['SEND'] LOAD_ATTR = opmap['LOAD_ATTR'] LOAD_SUPER_ATTR = opmap['LOAD_SUPER_ATTR'] +CALL = opmap['CALL'] +CALL_KW = opmap['CALL_KW'] +CALL_FUNCTION_EX = opmap['CALL_FUNCTION_EX'] +_CALL_AWAITED = 0x80 CALL_INTRINSIC_1 = opmap['CALL_INTRINSIC_1'] CALL_INTRINSIC_2 = opmap['CALL_INTRINSIC_2'] LOAD_COMMON_CONSTANT = opmap['LOAD_COMMON_CONSTANT'] @@ -642,6 +646,13 @@ def get_argval_argrepr(self, op, arg, offset): if arg & (1<', ['a test_stack_eager_task'], []]]) + async def test_capture_call_graph_through_generator_based_coroutine(self): + # A native coroutine driven by a generator-based coroutine + # (yield from) keeps the frames of its callers. + stack = None + + async def native(): + nonlocal stack + graph = asyncio.capture_call_graph() + stack = [entry.frame.f_code.co_name for entry in graph.call_stack] + + @types.coroutine + def legacy(): + return (yield from native()) + + async def main(): + await legacy() + + await main() + self.assertEqual(stack[:3], ['native', 'legacy', 'main']) + @unittest.skipIf( not hasattr(asyncio.futures, "_c_future_add_to_awaited_by"), diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index c5900da09501b3..3d88e1179845de 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1,6 +1,8 @@ import ast import contextlib import copy +import builtins +import gc import inspect import pickle import sys @@ -2594,6 +2596,648 @@ def test_unawaited_warning_during_shutdown(self): assert_python_ok("-c", code) +@support.cpython_only +class LazyCoroutineTest(unittest.TestCase): + """A coroutine that is awaited right away (``await f()``) runs in + the calling thread's frame and only gets a coroutine object when + it first suspends. This must not be observable, except that no + coroutine object exists before the first suspension.""" + + @staticmethod + def coroutines_for(code): + return [o for o in gc.get_objects() + if type(o) is types.CoroutineType and o.cr_code is code] + + def test_no_object_before_first_suspension(self): + seen = [] + + async def inner(): + seen.append(len(self.coroutines_for(inner.__code__))) + await AsyncYield(1) + seen.append(len(self.coroutines_for(inner.__code__))) + return 42 + + async def outer(): + return await inner() + + self.assertEqual(run_async(outer()), ([1], 42)) + self.assertEqual(seen, [0, 1]) + + def test_return_without_suspending(self): + async def inner(x): + return x + 1 + + async def outer(): + total = 0 + for i in range(100): + total += await inner(i) + return total + + self.assertEqual(run_async(outer()), ([], sum(range(1, 101)))) + + def test_nested_chain(self): + async def chain(n): + if n == 0: + await AsyncYield("bottom") + return 0 + return 1 + await chain(n - 1) + + self.assertEqual(run_async(chain(20)), (["bottom"], 20)) + + def test_exception_before_first_suspension(self): + async def inner(): + raise ValueError("boom") + + async def outer(): + try: + await inner() + except ValueError as exc: + tb = exc.__traceback__ + names = [] + while tb is not None: + names.append(tb.tb_frame.f_code.co_name) + tb = tb.tb_next + return names + + self.assertEqual(run_async(outer()), ([], ["outer", "inner"])) + + def test_stopiteration_before_first_suspension(self): + async def inner(): + raise StopIteration + + async def outer(): + await inner() + + with self.assertRaisesRegex(RuntimeError, + "coroutine raised StopIteration"): + run_async(outer()) + + def test_frame_object(self): + async def inner(): + x = 1 + frame = sys._getframe() + self.assertIsNone(frame.f_generator) + self.assertEqual(frame.f_locals["x"], 1) + self.assertEqual(frame.f_back.f_code.co_name, "outer") + await AsyncYield(1) + coro = frame.f_generator + self.assertIsInstance(coro, types.CoroutineType) + self.assertIs(coro.cr_frame, frame) + self.assertIs(coro.cr_code, inner.__code__) + self.assertEqual(frame.f_locals["x"], 1) + self.assertEqual(frame.f_back.f_code.co_name, "outer") + x = 2 + return frame + + async def outer(): + return await inner() + + _, frame = run_async(outer()) + self.assertIsNone(frame.f_generator) # finished + self.assertEqual(frame.f_locals["x"], 2) + + def test_exc_info_suspend_inside_handler(self): + async def inner(): + try: + raise KeyError + except KeyError: + await AsyncYield(1) + self.assertIsInstance(sys.exc_info()[1], KeyError) + await AsyncYield(2) + self.assertIsInstance(sys.exc_info()[1], KeyError) + # The caller's context is visible after the handler. + self.assertIsInstance(sys.exc_info()[1], TypeError) + await AsyncYield(3) + self.assertIsInstance(sys.exc_info()[1], TypeError) + return "ok" + + async def outer(): + try: + raise TypeError + except TypeError: + result = await inner() + self.assertIsInstance(sys.exc_info()[1], TypeError) + self.assertIsNone(sys.exc_info()[1]) + return result + + self.assertEqual(run_async(outer()), ([1, 2, 3], "ok")) + + def test_exc_info_handler_without_suspending(self): + async def inner(): + try: + raise KeyError + except KeyError: + self.assertIsInstance(sys.exc_info()[1], KeyError) + self.assertIsInstance(sys.exc_info()[1], TypeError) + return "ok" + + async def outer(): + try: + raise TypeError + except TypeError: + result = await inner() + self.assertIsInstance(sys.exc_info()[1], TypeError) + self.assertIsNone(sys.exc_info()[1]) + return result + + self.assertEqual(run_async(outer()), ([], "ok")) + + def test_throw_into_materialized_coroutine(self): + async def inner(): + try: + await AsyncYield(1) + except ZeroDivisionError: + return "caught" + return "not caught" + + async def outer(): + return await inner() + + coro = outer() + self.assertEqual(coro.send(None), 1) + with self.assertRaises(StopIteration) as cm: + coro.throw(ZeroDivisionError) + self.assertEqual(cm.exception.value, "caught") + + def test_close_materialized_coroutine(self): + closed = [] + + async def inner(): + try: + await AsyncYield(1) + finally: + closed.append(True) + + async def outer(): + await inner() + + coro = outer() + self.assertEqual(coro.send(None), 1) + coro.close() + self.assertEqual(closed, [True]) + + def test_async_with(self): + log = [] + + class CM: + async def __aenter__(self): + log.append("enter") + return self + + async def __aexit__(self, *args): + log.append("exit") + return False + + async def outer(): + async with CM() as cm: + log.append(type(cm).__name__) + await AsyncYield(1) + return "done" + + self.assertEqual(run_async(outer()), ([1], "done")) + self.assertEqual(log, ["enter", "CM", "exit"]) + + def test_async_comprehension(self): + async def inner(x): + return x * 2 + + async def outer(): + return [await inner(x) for x in range(5)] + + self.assertEqual(run_async(outer()), ([], [0, 2, 4, 6, 8])) + + def test_origin_tracking(self): + orig_depth = sys.get_coroutine_origin_tracking_depth() + + async def inner(): + await AsyncYield(1) + return sys._getframe().f_generator.cr_origin + + async def outer(): + coro = inner() + eager = (await coro)[0] + lazy = (await inner())[0] + return eager, lazy + + sys.set_coroutine_origin_tracking_depth(2) + try: + _, (eager, lazy) = run_async(outer()) + finally: + sys.set_coroutine_origin_tracking_depth(orig_depth) + # Origin tracking disables the lazy path: the origin is the + # call, not the await, and is captured at creation. + self.assertEqual(eager[0], __file__) + self.assertEqual(eager[2], "outer") + self.assertEqual(lazy[0], __file__) + self.assertEqual(lazy[2], "outer") + self.assertEqual(lazy[1], eager[1] + 2) + + def test_origin_tracking_multiline_await(self): + orig_depth = sys.get_coroutine_origin_tracking_depth() + + async def inner(): + await AsyncYield(1) + return sys._getframe().f_generator.cr_origin + + async def outer(): + origin = await ( + inner() + ) + return origin[0][1] - outer.__code__.co_firstlineno + + sys.set_coroutine_origin_tracking_depth(1) + try: + _, line = run_async(outer()) + finally: + sys.set_coroutine_origin_tracking_depth(orig_depth) + # The line of the call, not of the await. + self.assertEqual(line, 2) + + def test_settrace_events(self): + async def inner(): + await AsyncYield(1) + return 1 + + async def eager(): + coro = inner() + return await coro + + async def lazy(): + return await inner() + + def trace(coro_fn): + events = [] + + def tracer(frame, event, arg): + if frame.f_code is inner.__code__: + events.append((event, frame.f_lineno - frame.f_code.co_firstlineno)) + return tracer + + sys.settrace(tracer) + try: + result = run_async(coro_fn()) + finally: + sys.settrace(None) + return events, result + + self.assertEqual(trace(lazy), trace(eager)) + + def test_monitoring_events(self): + async def inner(): + await AsyncYield(1) + return 1 + + async def eager(): + coro = inner() + return await coro + + async def lazy(): + return await inner() + + E = sys.monitoring.events + # No LINE events: eager() has one more line. + names = {E.PY_START: "start", E.PY_RESUME: "resume", + E.PY_RETURN: "return", E.PY_YIELD: "yield", + E.STOP_ITERATION: "stop_iteration"} + codes = {inner.__code__: "inner", eager.__code__: "outer", + lazy.__code__: "outer"} + + def monitor(coro_fn): + events = [] + tool = sys.monitoring.DEBUGGER_ID + sys.monitoring.use_tool_id(tool, "test") + try: + for event, name in names.items(): + def callback(code, offset, *args, name=name): + if code in codes: + events.append( + (codes[code], name, type(args[-1]) if args else None)) + sys.monitoring.register_callback(tool, event, callback) + sys.monitoring.set_events(tool, sum(names)) + result = run_async(coro_fn()) + finally: + sys.monitoring.set_events(tool, 0) + sys.monitoring.free_tool_id(tool) + return events, result + + self.assertEqual(monitor(lazy), monitor(eager)) + self.assertIn(("outer", "stop_iteration", StopIteration), + monitor(lazy)[0]) + + def test_stop_iteration_event_without_suspending(self): + async def inner(): + return 42 + + async def outer(): + return await inner() + + E = sys.monitoring.events + events = [] + tool = sys.monitoring.DEBUGGER_ID + sys.monitoring.use_tool_id(tool, "test") + try: + def callback(code, offset, exc): + if code is outer.__code__: + events.append(exc.value) + sys.monitoring.register_callback(tool, E.STOP_ITERATION, callback) + sys.monitoring.set_events(tool, E.STOP_ITERATION) + self.assertEqual(run_async(outer()), ([], 42)) + finally: + sys.monitoring.set_events(tool, 0) + sys.monitoring.free_tool_id(tool) + self.assertEqual(events, [42]) + + def test_exc_state_after_suspending_inside_handler(self): + # The coroutine's own exception state must not keep the + # caller's exception once its handler is done. + async def inner(): + try: + raise KeyError + except KeyError: + await AsyncYield(1) + await AsyncYield(2) + return sys.exc_info()[1] + + async def outer(): + try: + raise TypeError + except TypeError: + return await inner() + + coro = outer() + coro.send(None) + inner_coro = coro.cr_await + inner_coro.send(None) + self.assertNotIn(TypeError, + map(type, gc.get_referents(inner_coro))) + # Resumed directly, not through outer: no handled exception. + with self.assertRaises(StopIteration) as cm: + inner_coro.send(None) + self.assertIsNone(cm.exception.value) + coro.close() + + def test_throw_into_coroutine_suspended_inside_handler(self): + async def inner(): + try: + raise KeyError + except KeyError: + try: + await AsyncYield(1) + except ZeroDivisionError: + pass + return sys.exception() + + async def outer(): + try: + raise TypeError + except TypeError: + result = await inner() + self.assertIsInstance(sys.exception(), TypeError) + return result + + coro = outer() + self.assertEqual(coro.send(None), 1) + with self.assertRaises(StopIteration) as cm: + coro.throw(ZeroDivisionError) + self.assertIsNone(cm.exception.value) + + def test_import_hook_locals(self): + seen = [] + orig_import = builtins.__import__ + + def hook(name, globals=None, locals=None, fromlist=(), level=0): + seen.append(locals) + return orig_import(name, globals, locals, fromlist, level) + + async def inner(): + import sys as _sys + return _sys + + async def outer(): + try: + raise TypeError + except TypeError: + return await inner() + + with support.swap_attr(builtins, "__import__", hook): + self.assertEqual(run_async(outer()), ([], sys)) + self.assertEqual(seen, [None]) + + def test_traceback_frame_keeps_no_caller_exception(self): + async def inner(): + raise ValueError + + async def outer(): + try: + raise TypeError + except TypeError: + try: + await inner() + except ValueError as exc: + return exc.__traceback__.tb_next.tb_frame + + _, frame = run_async(outer()) + self.assertIs(frame.f_code, inner.__code__) + self.assertNotIn(TypeError, map(type, gc.get_referents(frame))) + + def test_code_replaced_before_first_suspension(self): + # The coroutine object is created at the first suspension, so it + # must be sized for the code the frame runs, not func.__code__. + async def small(): + big, small # same free variables as big() + await AsyncYield(1) + + async def big(): + a = b = c = d = e = f = g = h = i = j = k = m = n = o = p = 1 + big.__code__ = small.__code__ + await AsyncYield(1) + return (a, b, c, d, e, f, g, h, i, j, k, m, n, o, p) + + async def outer(): + return await big() + + self.assertEqual(run_async(outer()), ([1], (1,) * 15)) + + def test_finished_frame_has_no_f_back(self): + async def inner(): + return sys._getframe() + + async def outer(): + return await inner() + + _, frame = run_async(outer()) + self.assertIs(frame.f_code, inner.__code__) + self.assertIsNone(frame.f_back) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_memory_error_at_first_suspension(self): + @types.coroutine + def suspend(): + yield 1 + + awaitable = suspend() + + async def inner(): + _testcapi.set_nomemory(0, 1) + try: + await awaitable + finally: + _testcapi.remove_mem_hooks() + return "resumed" + + async def outer(): + return await inner() + + coro = outer() + try: + with self.assertRaises(MemoryError): + coro.send(None) + finally: + _testcapi.remove_mem_hooks() + awaitable.close() + + def test_call_forms(self): + class C: + async def meth(self, x, *, y=0): + if y: + await AsyncYield(y) + return ("meth", x, y) + + async def func(*args, **kwargs): + if kwargs.get("suspend"): + await AsyncYield("f") + return (args, kwargs) + + obj = C() + bound = obj.meth + + async def outer(): + return [ + await obj.meth(1), + await obj.meth(2, y=3), + await bound(4), + await bound(5, y=6), + await func(7), + await func(*[8, 9]), + await func(**{"a": 10}), + await func(11, suspend=True), + ] + + for _ in range(3): # also with specialized calls + self.assertEqual(run_async(outer()), ( + [3, 6, "f"], + [("meth", 1, 0), ("meth", 2, 3), ("meth", 4, 0), + ("meth", 5, 6), ((7,), {}), ((8, 9), {}), + ((), {"a": 10}), ((11,), {"suspend": True})], + )) + + def test_send_specialized_for_other_awaitables(self): + # The SEND of an await site may have been specialized for another + # kind of awaitable before a coroutine function is awaited there. + async def agen(): + yield 1 + + class Iterator: + async def __anext__(self): + return 2 + + class SuspendingIterator: + async def __anext__(self): + await AsyncYield("s") + return 3 + + async def outer(it): + return await type(it).__anext__(it) + + def check(it, expected): + self.assertEqual(run_async(outer(it)), expected) + + for _ in range(3): + for _ in range(20): + check(agen(), ([], 1)) + check(Iterator(), ([], 2)) + check(SuspendingIterator(), (["s"], 3)) + + def test_traceback_like_eager_coroutine(self): + async def inner(): + raise ValueError + + async def outer(): + await inner() + + def format_tb(): + try: + run_async(outer()) + except ValueError: + return traceback.format_exc() + + lazy = format_tb() + # Origin tracking makes awaited calls create the coroutine eagerly. + old_depth = sys.get_coroutine_origin_tracking_depth() + sys.set_coroutine_origin_tracking_depth(1) + try: + eager = format_tb() + finally: + sys.set_coroutine_origin_tracking_depth(old_depth) + # The whole `await inner()` line, not just `inner()` with carets. + self.assertIn(" await inner()\n File", lazy) + self.assertEqual(lazy, eager) + + def test_traceback_multiline_await(self): + async def inner(): + raise ValueError + + async def outer(): + x = (await + inner()) + return x + + def format_tb(): + try: + run_async(outer()) + except ValueError: + return traceback.format_exc() + + lazy = format_tb() + old_depth = sys.get_coroutine_origin_tracking_depth() + sys.set_coroutine_origin_tracking_depth(1) + try: + eager = format_tb() + finally: + sys.set_coroutine_origin_tracking_depth(old_depth) + self.assertEqual(lazy, eager) + + def test_instrumentation_enabled_before_first_suspension(self): + # END_AWAITED_CALL must run right after the body suspends, before + # anything else, even if LINE or INSTRUCTION events get enabled + # while the body runs. + E = sys.monitoring.events + tool = sys.monitoring.DEBUGGER_ID + + async def inner(events): + sys.monitoring.set_events(tool, events) + await AsyncYield("suspended") + return "done" + + async def outer(events): + x = (await + inner(events)) + return x + + for events in (E.LINE, E.INSTRUCTION, E.LINE | E.INSTRUCTION): + with self.subTest(events=events): + sys.monitoring.use_tool_id(tool, "test") + try: + for event in (E.LINE, E.INSTRUCTION): + sys.monitoring.register_callback( + tool, event, lambda *args: gc.collect()) + self.assertEqual(run_async(outer(events)), + (["suspended"], "done")) + finally: + sys.monitoring.set_events(tool, 0) + sys.monitoring.free_tool_id(tool) + + @support.cpython_only @unittest.skipIf(_testcapi is None, "requires _testcapi") class CAPITest(unittest.TestCase): diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 709fa374704f1f..d4242ee1e97265 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1314,6 +1314,25 @@ async def afunc(): self.assertNotIn("to", line) self.assertIn("from", line) + def test_awaited_call(self): + + async def afunc(): + await f(1) + await x + + instrs = list(dis.get_instructions(afunc)) + names = [i.opname for i in instrs] + start = names.index('END_AWAITED_CALL') + self.assertEqual(names[start - 2:start + 7], [ + 'LOAD_SMALL_INT', 'CALL', 'END_AWAITED_CALL', 'GET_AWAITABLE', + 'PUSH_NULL', 'LOAD_COMMON_CONSTANT', 'SEND', 'YIELD_VALUE', + 'RESUME']) + self.assertEqual(instrs[start].jump_target, + instrs[start + 5].offset) + self.assertEqual(instrs[start - 1].argrepr, '1 + awaited') + # Only `await ` gets END_AWAITED_CALL. + self.assertEqual(names.count('END_AWAITED_CALL'), 1) + @staticmethod def code_quicken(f): diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 5f7c15f93a8332..bc05669de820bc 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -548,6 +548,57 @@ def count_args(self, *args): MyClass.__init__.__code__ = count_args.__code__ instantiate() + @requires_jit_disabled + @requires_specialization + def test_awaited_calls(self): + async def exact(x): + return sys._getframe().f_generator is None + + async def general(x, y=None): + return sys._getframe().f_generator is None + + class C: + async def exact(self, x): + return sys._getframe().f_generator is None + + async def general(self, x, y=None): + return sys._getframe().f_generator is None + + c = C() + bound_exact = c.exact + bound_general = c.general + + async def outer(): + return [ + await exact(1), + await general(1), + await bound_exact(1), + await bound_general(1), + await exact(x=1), + await bound_exact(x=1), + ] + + def run(): + coro = outer() + try: + coro.send(None) + except StopIteration as exc: + return exc.value + self.fail("outer() suspended") + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + # All coroutines run without a coroutine object. + self.assertEqual(run(), [True] * 6) + for opname in ("CALL_PY_EXACT_ARGS_AWAITED", + "CALL_PY_GENERAL_AWAITED", + "CALL_BOUND_METHOD_EXACT_ARGS_AWAITED", + "CALL_BOUND_METHOD_GENERAL_AWAITED", + "CALL_KW_PY_AWAITED", + "CALL_KW_BOUND_METHOD_AWAITED"): + self.assert_specialized(outer, opname) + self.assert_no_opcode(outer, "CALL_PY_EXACT_ARGS") + self.assertEqual(run(), [True] * 6) + @requires_jit_disabled @requires_specialization def test_push_init_frame_fails(self): @@ -1630,8 +1681,15 @@ def run_async(coro): except StopIteration: break + class Suspend: + def __await__(self): + yield + class CM: async def __aenter__(self): + # A coroutine that is awaited right away and never + # suspends doesn't execute the caller's SEND at all. + await Suspend() return self async def __aexit__(self, *exc): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-27-12-00-00.gh-issue-161170.lazyCo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-27-12-00-00.gh-issue-161170.lazyCo.rst new file mode 100644 index 00000000000000..ac0dc2f37e8181 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-27-12-00-00.gh-issue-161170.lazyCo.rst @@ -0,0 +1,3 @@ +Make ``await`` of a coroutine that completes without suspending about twice as +fast by running it without creating a coroutine object. Until such a coroutine +first suspends, :attr:`frame.f_generator` is ``None``. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index cfa8faa15f4fae..ba66fb563983b2 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -1831,8 +1831,8 @@ _PyStackRef res; // _SPECIALIZE_CALL { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg & ~CALL_AWAITED)]; + callable = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION @@ -1840,7 +1840,7 @@ next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, (oparg & ~CALL_AWAITED) + !PyStackRef_IsNull(self_or_null)); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } @@ -1858,8 +1858,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-1 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -1868,9 +1868,9 @@ } // _DO_CALL { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg & ~CALL_AWAITED)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = oparg & ~CALL_AWAITED; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -1882,8 +1882,8 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-1 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( @@ -1891,16 +1891,19 @@ arguments, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & CALL_AWAITED) && opcode != INSTRUMENTED_CALL) { + MAYBE_START_LAZY_CORO(new_frame); + } frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-1 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( @@ -1914,7 +1917,7 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -1922,8 +1925,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = res; + stack_pointer += -1 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2233,6 +2236,164 @@ DISPATCH(); } + TARGET(CALL_BOUND_METHOD_EXACT_ARGS_AWAITED) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BOUND_METHOD_EXACT_ARGS_AWAITED; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS_AWAITED); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + // _CLEAR_CALL_AWAITED + { + oparg &= ~CALL_AWAITED; + } + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_FUNCTION_VERSION + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _START_LAZY_CORO + { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(CALL_BOUND_METHOD_GENERAL) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_BOUND_METHOD_GENERAL; @@ -2369,6 +2530,150 @@ DISPATCH(); } + TARGET(CALL_BOUND_METHOD_GENERAL_AWAITED) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BOUND_METHOD_GENERAL_AWAITED; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL_AWAITED); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + // _CLEAR_CALL_AWAITED + { + oparg &= ~CALL_AWAITED; + } + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_METHOD_VERSION + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _EXPAND_METHOD + { + self_or_null = null; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _START_LAZY_CORO + { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(CALL_BUILTIN_CLASS) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_BUILTIN_CLASS; @@ -2972,6 +3277,13 @@ } ex_frame = PyStackRef_Wrap(new_frame); } + // _MAYBE_LAZY_CORO + { + new_frame = ex_frame; + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -2983,7 +3295,6 @@ } // _PUSH_FRAME { - new_frame = ex_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3150,6 +3461,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if (oparg & 1) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = 2u ; DISPATCH_INLINED(new_frame); @@ -3394,8 +3708,8 @@ _PyStackRef res; // _SPECIALIZE_CALL_KW { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + self_or_null = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; + callable = stack_pointer[-3 - (oparg & ~CALL_AWAITED)]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION @@ -3403,7 +3717,7 @@ next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_CallKw(callable, next_instr, (oparg & ~CALL_AWAITED) + !PyStackRef_IsNull(self_or_null)); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } @@ -3421,8 +3735,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -3432,10 +3746,10 @@ // _DO_CALL_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg & ~CALL_AWAITED)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; + int total_args = oparg & ~CALL_AWAITED; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3448,8 +3762,8 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( @@ -3457,7 +3771,7 @@ arguments, positional_args, kwnames_o, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -3466,47 +3780,189 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & CALL_AWAITED) && opcode != INSTRUMENTED_CALL_KW) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } + stack_pointer[-3 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( + callable, + arguments, + total_args, + kwnames, + opcode == INSTRUMENTED_CALL_KW, + frame, + this_instr, + tstate); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + stack_pointer += -3 - (oparg & ~CALL_AWAITED); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-3 - (oparg & ~CALL_AWAITED)] = res; + stack_pointer += -2 - (oparg & ~CALL_AWAITED); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CALL_KW_BOUND_METHOD) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_BOUND_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_METHOD_VERSION_KW + { + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _EXPAND_METHOD_KW + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + _PyStackRef callable_s = callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( - callable, - arguments, - total_args, - kwnames, - opcode == INSTRUMENTED_CALL_KW, - frame, - this_instr, - tstate); + PyStackRef_CLOSE(callable_s); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - if (res_o == NULL) { - stack_pointer += -3 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } - TARGET(CALL_KW_BOUND_METHOD) { + TARGET(CALL_KW_BOUND_METHOD_AWAITED) { #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW_BOUND_METHOD; + int opcode = CALL_KW_BOUND_METHOD_AWAITED; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD_AWAITED); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -3514,6 +3970,10 @@ _PyStackRef *args; _PyStackRef kwnames; _PyStackRef new_frame; + // _CLEAR_CALL_AWAITED + { + oparg &= ~CALL_AWAITED; + } /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -3605,6 +4065,10 @@ } new_frame = PyStackRef_Wrap(temp); } + // _START_LAZY_CORO + { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -3833,6 +4297,132 @@ DISPATCH(); } + TARGET(CALL_KW_PY_AWAITED) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_PY_AWAITED; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY_AWAITED); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + // _CLEAR_CALL_AWAITED + { + oparg &= ~CALL_AWAITED; + } + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _START_LAZY_CORO + { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(CALL_LEN) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_LEN; @@ -4590,39 +5180,162 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } - res = PyStackRef_FromPyObjectSteal(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_PY_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_PY_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); } - // _CHECK_PERIODIC_AT_END + // _SAVE_RETURN_OFFSET { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - int err = check_periodics_at_end(tstate, frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); } DISPATCH(); } - TARGET(CALL_PY_EXACT_ARGS) { + TARGET(CALL_PY_EXACT_ARGS_AWAITED) { #if _Py_TAIL_CALL_INTERP - int opcode = CALL_PY_EXACT_ARGS; + int opcode = CALL_PY_EXACT_ARGS_AWAITED; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS_AWAITED); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef new_frame; + // _CLEAR_CALL_AWAITED + { + oparg &= ~CALL_AWAITED; + } /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -4694,6 +5407,10 @@ } new_frame = PyStackRef_Wrap(pushed_frame); } + // _START_LAZY_CORO + { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -4834,6 +5551,121 @@ DISPATCH(); } + TARGET(CALL_PY_GENERAL_AWAITED) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_PY_GENERAL_AWAITED; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL_AWAITED); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + // _CLEAR_CALL_AWAITED + { + oparg &= ~CALL_AWAITED; + } + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _START_LAZY_CORO + { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(CALL_STR_1) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_STR_1; @@ -6119,6 +6951,74 @@ DISPATCH(); } + TARGET(END_AWAITED_CALL) { + #if _Py_TAIL_CALL_INTERP + int opcode = END_AWAITED_CALL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_AWAITED_CALL); + _PyStackRef lazy_outcome; + _PyStackRef res; + _PyStackRef value; + // _END_AWAITED_CALL + { + res = stack_pointer[-1]; + lazy_outcome = stack_pointer[-2]; + if (PyStackRef_IsTaggedInt(lazy_outcome)) { + lazy_outcome = PyStackRef_NULL; + _Py_CODEUNIT *send = next_instr + oparg - 1 - INLINE_CACHE_ENTRIES_SEND; + stack_pointer[-2] = lazy_outcome; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(_Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), + (int)(send - _PyFrame_GetBytecode(frame))).op.code == SEND); + _PyFrame_StackPointerInvalidate(frame); + _Py_CODEUNIT *end_send = next_instr + oparg + send->op.arg; + if (end_send->op.code != END_SEND) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = monitor_stop_iteration(tstate, frame, end_send, + PyStackRef_AsPyObjectBorrow(res)); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + JUMPBY(oparg + send->op.arg + 1); + } + else if (!PyStackRef_IsNull(lazy_outcome)) { + _PyInterpreterFrame *lazy_frame = PyStackRef_Unwrap(lazy_outcome); + lazy_outcome = PyStackRef_NULL; + stack_pointer[-2] = lazy_outcome; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = _PyEval_MaterializeLazyCoro(tstate, lazy_frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef v = res; + stack_pointer[-2] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + stack_pointer[-1] = PyStackRef_NULL; + stack_pointer[0] = v; + stack_pointer += 1; + JUMPBY(oparg); + DISPATCH(); + } + } + // _POP_LAZY_OUTCOME + { + assert(PyStackRef_IsNull(lazy_outcome)); + value = res; + } + stack_pointer[-2] = value; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + TARGET(END_FOR) { #if _Py_TAIL_CALL_INTERP int opcode = END_FOR; @@ -7111,8 +8011,8 @@ /* Skip 3 cache entries */ // _MAYBE_EXPAND_METHOD { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg & ~CALL_AWAITED)]; + callable = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -7120,8 +8020,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-1 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -7130,7 +8030,7 @@ } // _MONITOR_CALL { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg & ~CALL_AWAITED)]; maybe_self = self_or_null; func = callable; int is_meth = !PyStackRef_IsNull(maybe_self); @@ -7139,14 +8039,14 @@ if (is_meth) { arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); } - else if (oparg) { + else if (oparg & ~CALL_AWAITED) { arg0 = PyStackRef_AsPyObjectBorrow(args[0]); } else { arg0 = &_PyInstrumentation_MISSING; } - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = maybe_self; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = func; + stack_pointer[-1 - (oparg & ~CALL_AWAITED)] = maybe_self; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( @@ -7160,11 +8060,11 @@ } // _DO_CALL { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg & ~CALL_AWAITED)]; + self_or_null = stack_pointer[-1 - (oparg & ~CALL_AWAITED)]; + callable = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = oparg & ~CALL_AWAITED; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -7183,11 +8083,14 @@ arguments, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & CALL_AWAITED) && opcode != INSTRUMENTED_CALL) { + MAYBE_START_LAZY_CORO(new_frame); + } frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } @@ -7204,7 +8107,7 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -7212,8 +8115,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = res; + stack_pointer += -1 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -7355,6 +8258,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if (oparg & 1) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = 2u ; DISPATCH_INLINED(new_frame); @@ -7428,8 +8334,8 @@ /* Skip 2 cache entries */ // _MAYBE_EXPAND_METHOD_KW { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + self_or_null = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; + callable = stack_pointer[-3 - (oparg & ~CALL_AWAITED)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -7437,8 +8343,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -7447,7 +8353,7 @@ } // _MONITOR_CALL_KW { - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg & ~CALL_AWAITED)]; int is_meth = !PyStackRef_IsNull(self_or_null); PyObject *arg; if (is_meth) { @@ -7460,8 +8366,8 @@ arg = &_PyInstrumentation_MISSING; } PyObject *function = PyStackRef_AsPyObjectBorrow(callable); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( @@ -7475,10 +8381,10 @@ // _DO_CALL_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg & ~CALL_AWAITED)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; + int total_args = oparg & ~CALL_AWAITED; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -7498,7 +8404,7 @@ arguments, positional_args, kwnames_o, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -7507,6 +8413,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & CALL_AWAITED) && opcode != INSTRUMENTED_CALL_KW) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); @@ -7524,14 +8433,14 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-3 - (oparg & ~CALL_AWAITED)] = res; + stack_pointer += -2 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8283,6 +9192,9 @@ _PyStackRef val; _PyStackRef value; _PyStackRef retval; + // _CHECK_LAZY_CORO + { + } // _YIELD_VALUE_EVENT { val = stack_pointer[-1]; @@ -8316,36 +9228,50 @@ retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + int lazy = frame->lazy_coro; + PyGenObject *gen = lazy ? NULL : _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); + assert(!lazy || oparg == 1); _PyStackRef temp = retval; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); DTRACE_FUNCTION_RETURN(); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; + _PyErr_StackItem *exc_info = lazy ? tstate->exc_info : &gen->gi_exc_state; + tstate->exc_info = exc_info->previous_item; + exc_info->previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; - FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + if (!lazy) { + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + } assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_Wrap(gen_frame); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -11442,32 +12368,57 @@ _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - JUMP_TO_LABEL(error); + _PyErr_StackItem *exc_info = NULL; + #if TIER_ONE + if (frame->lazy_coro) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + _PyFrame_StackPointerInvalidate(frame); + if (exc_info == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + } + } + #endif + if (exc_info != NULL) { + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + res = PyStackRef_None; + } + else { + frame->lazy_coro = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -13574,6 +14525,9 @@ opcode = YIELD_VALUE; _PyStackRef value; _PyStackRef retval; + // _CHECK_LAZY_CORO + { + } // _MAKE_HEAP_SAFE { value = stack_pointer[-1]; @@ -13584,36 +14538,50 @@ retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + int lazy = frame->lazy_coro; + PyGenObject *gen = lazy ? NULL : _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); + assert(!lazy || oparg == 1); _PyStackRef temp = retval; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); DTRACE_FUNCTION_RETURN(); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; + _PyErr_StackItem *exc_info = lazy ? tstate->exc_info : &gen->gi_exc_state; + tstate->exc_info = exc_info->previous_item; + exc_info->previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; - FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + if (!lazy) { + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + } assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_Wrap(gen_frame); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -13766,6 +14734,9 @@ JUMP_TO_LABEL(error); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; + if (dying->lazy_coro) { + frame->instr_ptr += frame->return_offset; + } _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index 91b424773224e4..b796c9b8af1b29 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -4,7 +4,6 @@ static void *opcode_targets_table[256] = { &&TARGET_BINARY_SLICE, &&TARGET_BUILD_TEMPLATE, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, - &&TARGET_CALL_FUNCTION_EX, &&TARGET_CHECK_EG_MATCH, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CLEANUP_THROW, @@ -17,8 +16,8 @@ static void *opcode_targets_table[256] = { &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, &&TARGET_GET_LEN, - &&TARGET_RESERVED, &&TARGET_INTERPRETER_EXIT, + &&TARGET_RESERVED, &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_LOCALS, &&TARGET_MAKE_FUNCTION, @@ -51,6 +50,7 @@ static void *opcode_targets_table[256] = { &&TARGET_BUILD_STRING, &&TARGET_BUILD_TUPLE, &&TARGET_CALL, + &&TARGET_CALL_FUNCTION_EX, &&TARGET_CALL_INTRINSIC_1, &&TARGET_CALL_INTRINSIC_2, &&TARGET_CALL_KW, @@ -64,6 +64,7 @@ static void *opcode_targets_table[256] = { &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, &&TARGET_END_ASYNC_FOR, + &&TARGET_END_AWAITED_CALL, &&TARGET_EXTENDED_ARG, &&TARGET_FOR_ITER, &&TARGET_GET_AWAITABLE, @@ -127,7 +128,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -146,7 +146,9 @@ static void *opcode_targets_table[256] = { &&TARGET_BINARY_OP_SUBTRACT_INT, &&TARGET_CALL_ALLOC_AND_ENTER_INIT, &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS_AWAITED, &&TARGET_CALL_BOUND_METHOD_GENERAL, + &&TARGET_CALL_BOUND_METHOD_GENERAL_AWAITED, &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_CALL_BUILTIN_FAST, &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, @@ -155,8 +157,10 @@ static void *opcode_targets_table[256] = { &&TARGET_CALL_EX_PY, &&TARGET_CALL_ISINSTANCE, &&TARGET_CALL_KW_BOUND_METHOD, + &&TARGET_CALL_KW_BOUND_METHOD_AWAITED, &&TARGET_CALL_KW_NON_PY, &&TARGET_CALL_KW_PY, + &&TARGET_CALL_KW_PY_AWAITED, &&TARGET_CALL_LEN, &&TARGET_CALL_LIST_APPEND, &&TARGET_CALL_METHOD_DESCRIPTOR_FAST, @@ -165,7 +169,9 @@ static void *opcode_targets_table[256] = { &&TARGET_CALL_METHOD_DESCRIPTOR_O, &&TARGET_CALL_NON_PY_GENERAL, &&TARGET_CALL_PY_EXACT_ARGS, + &&TARGET_CALL_PY_EXACT_ARGS_AWAITED, &&TARGET_CALL_PY_GENERAL, + &&TARGET_CALL_PY_GENERAL_AWAITED, &&TARGET_CALL_STR_1, &&TARGET_CALL_TUPLE_1, &&TARGET_CALL_TYPE_1, @@ -227,12 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -376,7 +376,7 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -478,12 +478,12 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -561,7 +561,9 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CACHE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS_AWAITED(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_GENERAL_AWAITED(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); @@ -574,8 +576,10 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAM static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_BOUND_METHOD_AWAITED(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_PY_AWAITED(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); @@ -584,7 +588,9 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_EXACT_ARGS_AWAITED(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_GENERAL_AWAITED(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); @@ -607,6 +613,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_AWAITED_CALL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); @@ -804,7 +811,9 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [CALL] = _TAIL_CALL_CALL, [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT, [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS, + [CALL_BOUND_METHOD_EXACT_ARGS_AWAITED] = _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS_AWAITED, [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_CALL_BOUND_METHOD_GENERAL, + [CALL_BOUND_METHOD_GENERAL_AWAITED] = _TAIL_CALL_CALL_BOUND_METHOD_GENERAL_AWAITED, [CALL_BUILTIN_CLASS] = _TAIL_CALL_CALL_BUILTIN_CLASS, [CALL_BUILTIN_FAST] = _TAIL_CALL_CALL_BUILTIN_FAST, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS, @@ -817,8 +826,10 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [CALL_ISINSTANCE] = _TAIL_CALL_CALL_ISINSTANCE, [CALL_KW] = _TAIL_CALL_CALL_KW, [CALL_KW_BOUND_METHOD] = _TAIL_CALL_CALL_KW_BOUND_METHOD, + [CALL_KW_BOUND_METHOD_AWAITED] = _TAIL_CALL_CALL_KW_BOUND_METHOD_AWAITED, [CALL_KW_NON_PY] = _TAIL_CALL_CALL_KW_NON_PY, [CALL_KW_PY] = _TAIL_CALL_CALL_KW_PY, + [CALL_KW_PY_AWAITED] = _TAIL_CALL_CALL_KW_PY_AWAITED, [CALL_LEN] = _TAIL_CALL_CALL_LEN, [CALL_LIST_APPEND] = _TAIL_CALL_CALL_LIST_APPEND, [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST, @@ -827,7 +838,9 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O, [CALL_NON_PY_GENERAL] = _TAIL_CALL_CALL_NON_PY_GENERAL, [CALL_PY_EXACT_ARGS] = _TAIL_CALL_CALL_PY_EXACT_ARGS, + [CALL_PY_EXACT_ARGS_AWAITED] = _TAIL_CALL_CALL_PY_EXACT_ARGS_AWAITED, [CALL_PY_GENERAL] = _TAIL_CALL_CALL_PY_GENERAL, + [CALL_PY_GENERAL_AWAITED] = _TAIL_CALL_CALL_PY_GENERAL_AWAITED, [CALL_STR_1] = _TAIL_CALL_CALL_STR_1, [CALL_TUPLE_1] = _TAIL_CALL_CALL_TUPLE_1, [CALL_TYPE_1] = _TAIL_CALL_CALL_TYPE_1, @@ -850,6 +863,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [DICT_MERGE] = _TAIL_CALL_DICT_MERGE, [DICT_UPDATE] = _TAIL_CALL_DICT_UPDATE, [END_ASYNC_FOR] = _TAIL_CALL_END_ASYNC_FOR, + [END_AWAITED_CALL] = _TAIL_CALL_END_AWAITED_CALL, [END_FOR] = _TAIL_CALL_END_FOR, [END_SEND] = _TAIL_CALL_END_SEND, [ENTER_EXECUTOR] = _TAIL_CALL_ENTER_EXECUTOR, @@ -1005,7 +1019,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1016,12 +1029,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, - [220] = _TAIL_CALL_UNKNOWN_OPCODE, - [221] = _TAIL_CALL_UNKNOWN_OPCODE, - [222] = _TAIL_CALL_UNKNOWN_OPCODE, - [223] = _TAIL_CALL_UNKNOWN_OPCODE, - [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, [226] = _TAIL_CALL_UNKNOWN_OPCODE, [227] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1062,7 +1069,9 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [CALL] = _TAIL_CALL_TRACE_RECORD, [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_TRACE_RECORD, [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_EXACT_ARGS_AWAITED] = _TAIL_CALL_TRACE_RECORD, [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_GENERAL_AWAITED] = _TAIL_CALL_TRACE_RECORD, [CALL_BUILTIN_CLASS] = _TAIL_CALL_TRACE_RECORD, [CALL_BUILTIN_FAST] = _TAIL_CALL_TRACE_RECORD, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD, @@ -1075,8 +1084,10 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [CALL_ISINSTANCE] = _TAIL_CALL_TRACE_RECORD, [CALL_KW] = _TAIL_CALL_TRACE_RECORD, [CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_BOUND_METHOD_AWAITED] = _TAIL_CALL_TRACE_RECORD, [CALL_KW_NON_PY] = _TAIL_CALL_TRACE_RECORD, [CALL_KW_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_PY_AWAITED] = _TAIL_CALL_TRACE_RECORD, [CALL_LEN] = _TAIL_CALL_TRACE_RECORD, [CALL_LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_TRACE_RECORD, @@ -1085,7 +1096,9 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_TRACE_RECORD, [CALL_NON_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, [CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_EXACT_ARGS_AWAITED] = _TAIL_CALL_TRACE_RECORD, [CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_GENERAL_AWAITED] = _TAIL_CALL_TRACE_RECORD, [CALL_STR_1] = _TAIL_CALL_TRACE_RECORD, [CALL_TUPLE_1] = _TAIL_CALL_TRACE_RECORD, [CALL_TYPE_1] = _TAIL_CALL_TRACE_RECORD, @@ -1108,6 +1121,7 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [DICT_MERGE] = _TAIL_CALL_TRACE_RECORD, [DICT_UPDATE] = _TAIL_CALL_TRACE_RECORD, [END_ASYNC_FOR] = _TAIL_CALL_TRACE_RECORD, + [END_AWAITED_CALL] = _TAIL_CALL_TRACE_RECORD, [END_FOR] = _TAIL_CALL_TRACE_RECORD, [END_SEND] = _TAIL_CALL_TRACE_RECORD, [ENTER_EXECUTOR] = _TAIL_CALL_TRACE_RECORD, @@ -1263,7 +1277,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1274,12 +1287,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, - [220] = _TAIL_CALL_UNKNOWN_OPCODE, - [221] = _TAIL_CALL_UNKNOWN_OPCODE, - [222] = _TAIL_CALL_UNKNOWN_OPCODE, - [223] = _TAIL_CALL_UNKNOWN_OPCODE, - [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, [226] = _TAIL_CALL_UNKNOWN_OPCODE, [227] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Objects/genobject.c b/Objects/genobject.c index 82a1bb1ff42068..a818c093e045de 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1104,9 +1104,8 @@ PyTypeObject PyGen_Type = { }; static PyObject * -make_gen(PyTypeObject *type, PyFunctionObject *func) +make_gen(PyTypeObject *type, PyFunctionObject *func, PyCodeObject *code) { - PyCodeObject *code = (PyCodeObject *)func->func_code; int slots = _PyFrame_NumSlotsForCodeObject(code); PyGenObject *gen = PyObject_GC_NewVar(PyGenObject, type, slots); if (gen == NULL) { @@ -1127,18 +1126,44 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) return (PyObject *)gen; } +static PyObject * +make_coro(PyFunctionObject *func, PyCodeObject *code, + _PyInterpreterFrame *caller) +{ + PyObject *coro = make_gen(&PyCoro_Type, func, code); + if (!coro) { + return NULL; + } + PyThreadState *tstate = _PyThreadState_GET(); + int origin_depth = tstate->coroutine_origin_tracking_depth; + + if (origin_depth == 0) { + ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; + } else { + _PyInterpreterFrame *frame = _PyFrame_GetFirstComplete(caller); + PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, frame); + ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin; + if (!cr_origin) { + Py_DECREF(coro); + return NULL; + } + } + return coro; +} + PyObject * _Py_MakeCoro(PyFunctionObject *func) { - int coro_flags = ((PyCodeObject *)func->func_code)->co_flags & + PyCodeObject *code = (PyCodeObject *)func->func_code; + int coro_flags = code->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR); assert(coro_flags); if (coro_flags == CO_GENERATOR) { - return make_gen(&PyGen_Type, func); + return make_gen(&PyGen_Type, func, code); } if (coro_flags == CO_ASYNC_GENERATOR) { PyAsyncGenObject *ag; - ag = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func); + ag = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func, code); if (ag == NULL) { return NULL; } @@ -1150,28 +1175,33 @@ _Py_MakeCoro(PyFunctionObject *func) } assert (coro_flags == CO_COROUTINE); - PyObject *coro = make_gen(&PyCoro_Type, func); - if (!coro) { - return NULL; - } PyThreadState *tstate = _PyThreadState_GET(); - int origin_depth = tstate->coroutine_origin_tracking_depth; - - if (origin_depth == 0) { - ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; - } else { - _PyInterpreterFrame *frame = tstate->current_frame; - assert(frame); - assert(_PyFrame_IsIncomplete(frame)); - frame = _PyFrame_GetFirstComplete(frame->previous); - PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, frame); - ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin; - if (!cr_origin) { - Py_DECREF(coro); - return NULL; - } - } - return coro; + _PyInterpreterFrame *frame = tstate->current_frame; + assert(frame); + assert(_PyFrame_IsIncomplete(frame)); + return make_coro(func, code, frame->previous); +} + +/* Create the coroutine object for a coroutine whose body has been + * executing directly in `frame`, a frame owned by the thread, because + * its caller awaited the result immediately (see RETURN_GENERATOR in + * bytecodes.c). Called when the body suspends for the first time; + * `caller` is the frame that awaited it. + * The frame itself is moved into the coroutine by the caller. + * The body may have replaced func.__code__ in the meantime, so the + * coroutine is sized for the code the frame is actually running. */ +PyObject * +_PyCoro_MaterializeLazy(_PyInterpreterFrame *frame, _PyInterpreterFrame *caller) +{ + assert(frame->owner == FRAME_OWNED_BY_THREAD); + assert(frame->lazy_coro); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = + (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyCodeObject *code = _PyFrame_GetCode(frame); + assert((code->co_flags & + (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) == CO_COROUTINE); + return make_coro(func, code, caller); } static PyObject * diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index ebcb38b18dec90..50d9c2dbce5aba 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,19 +1,19 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 99,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0, - 0,0,0,0,0,115,188,0,0,0,128,0,0,0,90,0, - 77,7,69,0,112,0,90,0,77,7,69,4,112,1,89,2, - 31,0,78,1,50,1,0,0,0,0,0,0,29,0,89,2, - 31,0,78,2,89,0,76,6,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,50,2,0,0,0,0, - 0,0,29,0,89,1,76,8,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,31,0,50,0,0,0, - 0,0,0,0,78,3,42,26,0,0,0,0,0,0,0,0, - 0,0,112,5,78,6,67,0,0,0,65,24,0,0,112,6, - 89,2,31,0,78,4,89,6,12,0,78,5,89,5,89,6, - 42,26,0,0,0,0,0,0,0,0,0,0,12,0,48,4, - 50,1,0,0,0,0,0,0,29,0,71,26,0,0,9,0, - 28,0,77,7,33,0,41,7,105,0,0,0,0,122,18,70, + 0,0,0,0,0,115,188,0,0,0,128,0,0,0,91,0, + 78,7,70,0,113,0,91,0,78,7,70,4,113,1,90,2, + 30,0,79,1,49,1,0,0,0,0,0,0,28,0,90,2, + 30,0,79,2,90,0,77,6,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,49,2,0,0,0,0, + 0,0,28,0,90,1,77,8,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,30,0,49,0,0,0, + 0,0,0,0,79,3,41,26,0,0,0,0,0,0,0,0, + 0,0,113,5,79,6,68,0,0,0,66,24,0,0,113,6, + 90,2,30,0,79,4,90,6,11,0,79,5,90,5,90,6, + 41,26,0,0,0,0,0,0,0,0,0,0,11,0,47,4, + 49,1,0,0,0,0,0,0,28,0,72,26,0,0,8,0, + 27,0,78,7,32,0,41,7,105,0,0,0,0,122,18,70, 114,111,122,101,110,32,72,101,108,108,111,32,87,111,114,108, 100,122,8,115,121,115,46,97,114,103,118,218,6,99,111,110, 102,105,103,122,7,99,111,110,102,105,103,32,122,2,58,32, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index bd5debcd01e9bf..0ee00dd4664810 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1656,6 +1656,61 @@ dummy_func( iter = PyStackRef_FromPyObjectSteal(iter_o); } + // Follows an awaited call (see codegen_await()). `lazy_outcome` was + // pushed below the callable as NULL, and oparg jumps to the + // YIELD_VALUE of the SEND loop that follows. If the call ran a + // coroutine body without creating the coroutine object (see + // RETURN_GENERATOR), `lazy_outcome` tells how the body ended: + // - a tagged int if the body returned `res`: the SEND loop is + // skipped; + // - the body's frame if the body suspended, yielding `res`: the + // coroutine is created and the SEND loop entered at YIELD_VALUE. + replaced op(_END_AWAITED_CALL, (lazy_outcome, res -- lazy_outcome, res)) { + if (PyStackRef_IsTaggedInt(lazy_outcome)) { + lazy_outcome = PyStackRef_NULL; + _Py_CODEUNIT *send = next_instr + oparg - 1 - INLINE_CACHE_ENTRIES_SEND; + assert(_Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), + (int)(send - _PyFrame_GetBytecode(frame))).op.code == SEND); + _Py_CODEUNIT *end_send = next_instr + oparg + send->op.arg; + if (end_send->op.code != END_SEND) { + // Report what INSTRUMENTED_END_SEND would have. + int err = monitor_stop_iteration(tstate, frame, end_send, + PyStackRef_AsPyObjectBorrow(res)); + ERROR_IF(err); + } + JUMPBY(oparg + send->op.arg + 1); + } + else if (!PyStackRef_IsNull(lazy_outcome)) { + _PyInterpreterFrame *lazy_frame = PyStackRef_Unwrap(lazy_outcome); + lazy_outcome = PyStackRef_NULL; + PyGenObject *gen = _PyEval_MaterializeLazyCoro(tstate, lazy_frame); + ERROR_IF(gen == NULL); + // YIELD_VALUE expects (coroutine, NULL, value), as left by SEND. + _PyStackRef v = res; + DEAD(res); + DEAD(lazy_outcome); + stack_pointer[-2] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + stack_pointer[-1] = PyStackRef_NULL; + stack_pointer[0] = v; + stack_pointer += 1; + JUMPBY(oparg); + DISPATCH(); + } + } + + tier2 op(_GUARD_AWAITED_CALL_NOT_LAZY, (lazy_outcome, unused -- lazy_outcome, unused)) { + EXIT_IF(!PyStackRef_IsNull(lazy_outcome)); + } + + op(_POP_LAZY_OUTCOME, (lazy_outcome, res -- value)) { + assert(PyStackRef_IsNull(lazy_outcome)); + DEAD(lazy_outcome); + value = res; + DEAD(res); + } + + macro(END_AWAITED_CALL) = _END_AWAITED_CALL + _POP_LAZY_OUTCOME; + family(SEND, INLINE_CACHE_ENTRIES_SEND) = { SEND_GEN, SEND_VIRTUAL, @@ -1849,28 +1904,47 @@ dummy_func( EXIT_IF(what == PYGEN_RETURN); } + // A lazily started coroutine (see RETURN_GENERATOR) suspends through + // the caller's END_AWAITED_CALL, which tier two doesn't run. Tier two + // replaces this with _GUARD_NOT_LAZY_CORO. + replaced op(_CHECK_LAZY_CORO, (--)) { + } + + tier2 op(_GUARD_NOT_LAZY_CORO, (--)) { + EXIT_IF(frame->lazy_coro); + } + op(_YIELD_VALUE, (retval -- value)) { // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + // A lazily started coroutine has no coroutine object yet. Its + // frame and exception state item stay on the data stack, and the + // caller's END_AWAITED_CALL moves them into a new coroutine. + // Nothing runs in between: END_AWAITED_CALL is never instrumented. + int lazy = frame->lazy_coro; + PyGenObject *gen = lazy ? NULL : _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); + assert(!lazy || oparg == 1); _PyStackRef temp = retval; DEAD(retval); SAVE_STACK(); DTRACE_FUNCTION_RETURN(); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; + _PyErr_StackItem *exc_info = lazy ? tstate->exc_info : &gen->gi_exc_state; + tstate->exc_info = exc_info->previous_item; + exc_info->previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; - FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + if (!lazy) { + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + } /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) @@ -1878,16 +1952,28 @@ dummy_func( Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif RELOAD_STACK(); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + /* Hand the frame to the caller's END_AWAITED_CALL as the + * `lazy_outcome` below the awaited call. */ + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_Wrap(gen_frame); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); } macro(YIELD_VALUE) = + _CHECK_LAZY_CORO + _MAKE_HEAP_SAFE + _YIELD_VALUE; @@ -1907,6 +1993,7 @@ dummy_func( } macro(INSTRUMENTED_YIELD_VALUE) = + _CHECK_LAZY_CORO + _YIELD_VALUE_EVENT + _MAKE_HEAP_SAFE + _YIELD_VALUE; @@ -4314,13 +4401,17 @@ dummy_func( CALL_PY_GENERAL, CALL_BOUND_METHOD_GENERAL, CALL_NON_PY_GENERAL, + CALL_PY_EXACT_ARGS_AWAITED, + CALL_BOUND_METHOD_EXACT_ARGS_AWAITED, + CALL_PY_GENERAL_AWAITED, + CALL_BOUND_METHOD_GENERAL_AWAITED, }; - specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, unused[oparg & ~CALL_AWAITED] -- callable, self_or_null, unused[oparg & ~CALL_AWAITED])) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, (oparg & ~CALL_AWAITED) + !PyStackRef_IsNull(self_or_null)); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -4328,7 +4419,7 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } - op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, unused[oparg & ~CALL_AWAITED] -- callable, self_or_null, unused[oparg & ~CALL_AWAITED])) { if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -4341,11 +4432,11 @@ dummy_func( } // When calling Python, inline the call using DISPATCH_INLINED(). - op(_DO_CALL, (callable, self_or_null, args[oparg] -- res)) { + op(_DO_CALL, (callable, self_or_null, args[oparg & ~CALL_AWAITED] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); // oparg counts all of the args, but *not* self: - int total_args = oparg; + int total_args = oparg & ~CALL_AWAITED; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4372,6 +4463,9 @@ dummy_func( if (new_frame == NULL) { ERROR_NO_POP(); } + if ((oparg & CALL_AWAITED) && opcode != INSTRUMENTED_CALL) { + MAYBE_START_LAZY_CORO(new_frame); + } frame->return_offset = INSTRUCTION_SIZE; DISPATCH_INLINED(new_frame); } @@ -4391,14 +4485,14 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_MONITOR_CALL, (func, maybe_self, args[oparg] -- func, maybe_self, args[oparg])) { + op(_MONITOR_CALL, (func, maybe_self, args[oparg & ~CALL_AWAITED] -- func, maybe_self, args[oparg & ~CALL_AWAITED])) { int is_meth = !PyStackRef_IsNull(maybe_self); PyObject *function = PyStackRef_AsPyObjectBorrow(func); PyObject *arg0; if (is_meth) { arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); } - else if (oparg) { + else if (oparg & ~CALL_AWAITED) { arg0 = PyStackRef_AsPyObjectBorrow(args[0]); } else { @@ -4463,6 +4557,18 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; + macro(CALL_PY_GENERAL_AWAITED) = + _RECORD_CALLABLE + + _CLEAR_CALL_AWAITED + + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_FUNCTION_VERSION + + _CHECK_RECURSION_REMAINING + + _PY_FRAME_GENERAL + + _START_LAZY_CORO + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4496,6 +4602,20 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; + macro(CALL_BOUND_METHOD_GENERAL_AWAITED) = + _RECORD_BOUND_METHOD + + _CLEAR_CALL_AWAITED + + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_METHOD_VERSION + + _EXPAND_METHOD + + flush + // so that self is in the argument array + _CHECK_RECURSION_REMAINING + + _PY_FRAME_GENERAL + + _START_LAZY_CORO + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(PyFunction_Check(callable_o)); @@ -4616,6 +4736,37 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; + macro(CALL_BOUND_METHOD_EXACT_ARGS_AWAITED) = + _RECORD_BOUND_METHOD + + _CLEAR_CALL_AWAITED + + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + + _INIT_CALL_BOUND_METHOD_EXACT_ARGS + + flush + // In case the following deopt + _CHECK_FUNCTION_VERSION + + _CHECK_FUNCTION_EXACT_ARGS + + _CHECK_STACK_SPACE + + _CHECK_RECURSION_REMAINING + + _INIT_CALL_PY_EXACT_ARGS + + _START_LAZY_CORO + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + + // The specialized forms of awaited calls (see CALL_AWAITED) are the + // plain forms with CALL_AWAITED cleared from oparg first, and the + // coroutine body started lazily (see RETURN_GENERATOR). Tier two + // clears it when translating and starts the coroutine eagerly. + // After a deopt the generic CALL keeps the cleared oparg (and runs + // the call eagerly), so this may run again with it already cleared. + replaced op(_CLEAR_CALL_AWAITED, (--)) { + oparg &= ~CALL_AWAITED; + } + + replaced op(_START_LAZY_CORO, (new_frame -- new_frame)) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + macro(CALL_PY_EXACT_ARGS) = _RECORD_CALLABLE + unused/1 + // Skip over the counter @@ -4628,6 +4779,20 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; + macro(CALL_PY_EXACT_ARGS_AWAITED) = + _RECORD_CALLABLE + + _CLEAR_CALL_AWAITED + + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_FUNCTION_VERSION + + _CHECK_FUNCTION_EXACT_ARGS + + _CHECK_STACK_SPACE + + _CHECK_RECURSION_REMAINING + + _INIT_CALL_PY_EXACT_ARGS + + _START_LAZY_CORO + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + op(_GUARD_NOS_NULL, (null, unused -- null, unused)) { EXIT_IF(!PyStackRef_IsNull(null)); } @@ -5350,9 +5515,11 @@ dummy_func( CALL_KW_BOUND_METHOD, CALL_KW_PY, CALL_KW_NON_PY, + CALL_KW_PY_AWAITED, + CALL_KW_BOUND_METHOD_AWAITED, }; - op(_MONITOR_CALL_KW, (callable, self_or_null, args[oparg], unused -- callable, self_or_null, args[oparg], unused)) { + op(_MONITOR_CALL_KW, (callable, self_or_null, args[oparg & ~CALL_AWAITED], unused -- callable, self_or_null, args[oparg & ~CALL_AWAITED], unused)) { int is_meth = !PyStackRef_IsNull(self_or_null); PyObject *arg; if (is_meth) { @@ -5371,7 +5538,7 @@ dummy_func( ERROR_IF(err); } - op(_MAYBE_EXPAND_METHOD_KW, (callable, self_or_null, unused[oparg], unused -- callable, self_or_null, unused[oparg], unused)) { + op(_MAYBE_EXPAND_METHOD_KW, (callable, self_or_null, unused[oparg & ~CALL_AWAITED], unused -- callable, self_or_null, unused[oparg & ~CALL_AWAITED], unused)) { if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -5383,12 +5550,12 @@ dummy_func( } } - op(_DO_CALL_KW, (callable, self_or_null, args[oparg], kwnames -- res)) { + op(_DO_CALL_KW, (callable, self_or_null, args[oparg & ~CALL_AWAITED], kwnames -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); // oparg counts all of the args, but *not* self: - int total_args = oparg; + int total_args = oparg & ~CALL_AWAITED; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -5417,6 +5584,9 @@ dummy_func( if (new_frame == NULL) { ERROR_NO_POP(); } + if ((oparg & CALL_AWAITED) && opcode != INSTRUMENTED_CALL_KW) { + MAYBE_START_LAZY_CORO(new_frame); + } assert(INSTRUCTION_SIZE == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = INSTRUCTION_SIZE; DISPATCH_INLINED(new_frame); @@ -5485,6 +5655,18 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; + macro(CALL_KW_PY_AWAITED) = + _RECORD_CALLABLE_KW + + _CLEAR_CALL_AWAITED + + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_FUNCTION_VERSION_KW + + _CHECK_RECURSION_REMAINING + + _PY_FRAME_KW + + _START_LAZY_CORO + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + op(_CHECK_METHOD_VERSION_KW, (func_version/2, callable, null, unused[oparg], unused -- callable, null, unused[oparg], unused)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -5517,11 +5699,24 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable, self_or_null, unused[oparg], unused -- callable, self_or_null, unused[oparg], unused)) { + macro(CALL_KW_BOUND_METHOD_AWAITED) = + _RECORD_CALLABLE_KW + + _CLEAR_CALL_AWAITED + + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_METHOD_VERSION_KW + + _EXPAND_METHOD_KW + + flush + // so that self is in the argument array + _PY_FRAME_KW + + _START_LAZY_CORO + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + + specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable, self_or_null, unused[oparg & ~CALL_AWAITED], unused -- callable, self_or_null, unused[oparg & ~CALL_AWAITED], unused)) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_CallKw(callable, next_instr, (oparg & ~CALL_AWAITED) + !PyStackRef_IsNull(self_or_null)); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_KW); @@ -5662,6 +5857,9 @@ dummy_func( if (new_frame == NULL) { ERROR_NO_POP(); } + if (oparg & 1) { + MAYBE_START_LAZY_CORO(new_frame); + } assert(INSTRUCTION_SIZE == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = INSTRUCTION_SIZE; DISPATCH_INLINED(new_frame); @@ -5734,6 +5932,7 @@ dummy_func( _MAKE_CALLARGS_A_TUPLE + _CHECK_IS_PY_CALLABLE_EX + _PY_FRAME_EX + + _MAYBE_LAZY_CORO + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -5806,25 +6005,53 @@ dummy_func( inst(RETURN_GENERATOR, (-- res)) { assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - ERROR_IF(gen == NULL); - assert(STACK_LEVEL() <= 2); - SAVE_STACK(); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - RELOAD_STACK(); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); + _PyErr_StackItem *exc_info = NULL; + #if TIER_ONE + if (frame->lazy_coro) { + /* The body gets its own exception state, as SEND_GEN + * would have given it. It lives on the data stack right + * after this frame; clear_thread_frame() and _YIELD_VALUE + * pop it. */ + exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + if (exc_info == NULL) { + /* Out of memory: take the regular path. */ + _PyErr_Clear(tstate); + } + } + #endif + if (exc_info != NULL) { + /* An awaited call (see _MAYBE_LAZY_CORO): keep running the + * body in this frame, and only create the coroutine object + * if the body suspends (see _YIELD_VALUE). */ + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + /* The following POP_TOP discards this. */ + res = PyStackRef_None; + } + else { + frame->lazy_coro = 0; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + ERROR_IF(gen == NULL); + assert(STACK_LEVEL() <= 2); + SAVE_STACK(); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + RELOAD_STACK(); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + } } inst(BUILD_SLICE, (args[oparg] -- slice)) { @@ -6101,6 +6328,12 @@ dummy_func( DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, framesize)); } + replaced op(_MAYBE_LAZY_CORO, (new_frame -- new_frame)) { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } + op(_SAVE_RETURN_OFFSET, (--)) { #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); @@ -6392,15 +6625,15 @@ dummy_func( RECORD_VALUE(PyStackRef_AsPyObjectBorrow(value)); } - tier2 op(_RECORD_CALLABLE, (func, self, args[oparg] -- func, self, args[oparg])) { + tier2 op(_RECORD_CALLABLE, (func, self, args[oparg & ~CALL_AWAITED] -- func, self, args[oparg & ~CALL_AWAITED])) { RECORD_VALUE(PyStackRef_AsPyObjectBorrow(func)); } - tier2 op(_RECORD_CALLABLE_KW, (func, self, args[oparg], kwnames -- func, self, args[oparg], kwnames)) { + tier2 op(_RECORD_CALLABLE_KW, (func, self, args[oparg & ~CALL_AWAITED], kwnames -- func, self, args[oparg & ~CALL_AWAITED], kwnames)) { RECORD_VALUE(PyStackRef_AsPyObjectBorrow(func)); } - tier2 op(_RECORD_BOUND_METHOD, (callable, self, args[oparg] -- callable, self, args[oparg])) { + tier2 op(_RECORD_BOUND_METHOD, (callable, self, args[oparg & ~CALL_AWAITED] -- callable, self, args[oparg & ~CALL_AWAITED])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) == &PyMethod_Type) { RECORD_VALUE(callable_o); @@ -6520,6 +6753,11 @@ dummy_func( // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; + if (dying->lazy_coro) { + /* Raise from the caller's END_AWAITED_CALL, which has the + * location of the await. */ + frame->instr_ptr += frame->return_offset; + } _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { diff --git a/Python/ceval.c b/Python/ceval.c index ccc0edca2fc794..23bb1c5e127e76 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2005,10 +2005,9 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, return -1; } -static void -clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) +static inline void +clear_and_pop_thread_frame(PyThreadState *tstate, _PyInterpreterFrame *frame) { - assert(frame->owner == FRAME_OWNED_BY_THREAD); // Make sure that this is, indeed, the top frame. We can't check this in // _PyThreadState_PopFrame, since f_code is already cleared at that point: assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == @@ -2019,6 +2018,41 @@ clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) _PyThreadState_PopFrame(tstate, frame); } +/* A lazily started coroutine body finished without creating its coroutine + * object (see RETURN_GENERATOR). Kept out of clear_thread_frame(), which + * every return from a Python function goes through. */ +static Py_NO_INLINE void +clear_lazy_coro_frame(PyThreadState *tstate, _PyInterpreterFrame *frame) +{ + // Pop the body's exception state, restoring the caller's. + _PyErr_StackItem *exc_info = tstate->exc_info; + tstate->exc_info = exc_info->previous_item; + PyObject *exc_value = exc_info->exc_value; + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + frame->lazy_coro = 0; + /* Tell the caller's END_AWAITED_CALL through the `lazy_outcome` that the + * compiler pushed below the awaited call. */ + _PyInterpreterFrame *caller = frame->previous; + assert(PyStackRef_IsNull(caller->stackpointer[-1])); + caller->stackpointer[-1] = PyStackRef_TagInt(0); + // Like clear_gen_frame(): a frame object that outlives the + // coroutine must not keep the caller's frame alive via f_back. + frame->previous = NULL; + clear_and_pop_thread_frame(tstate, frame); + Py_XDECREF(exc_value); +} + +static void +clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) +{ + assert(frame->owner == FRAME_OWNED_BY_THREAD); + if (frame->lazy_coro) { + clear_lazy_coro_frame(tstate, frame); + return; + } + clear_and_pop_thread_frame(tstate, frame); +} + static void clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { @@ -3686,6 +3720,50 @@ _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name } } +/* END_AWAITED_CALL support. `frame` is the frame of a coroutine body that + * an awaited call ran without creating the coroutine object (see + * RETURN_GENERATOR). _YIELD_VALUE left it on the data stack, followed by the + * body's exception state item, when the body first suspended. Create the + * coroutine and move both into it. On failure, discard the frame and return + * NULL with an exception set. */ +PyGenObject * +_PyEval_MaterializeLazyCoro(PyThreadState *tstate, _PyInterpreterFrame *frame) +{ + assert(frame->owner == FRAME_OWNED_BY_THREAD); + assert(frame->lazy_coro); + assert(frame->previous == NULL); + const int item_slots = sizeof(_PyErr_StackItem) / sizeof(PyObject *); + _PyErr_StackItem *exc_info = + (_PyErr_StackItem *)(tstate->datastack_top - item_slots); + assert(exc_info->previous_item == NULL); + PyObject *exc_value = exc_info->exc_value; + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == + tstate->datastack_top); + PyGenObject *gen = (PyGenObject *)_PyCoro_MaterializeLazy( + frame, tstate->current_frame); + if (gen == NULL) { + frame->lazy_coro = 0; + clear_thread_frame(tstate, frame); + Py_XDECREF(exc_value); + return NULL; + } + gen->gi_exc_state.exc_value = exc_value; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyFrame_Copy(frame, gen_frame); + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + PyFrameObject *f = gen_frame->frame_obj; + if (f != NULL) { + Py_BEGIN_CRITICAL_SECTION(f); + f->f_frame = gen_frame; + Py_END_CRITICAL_SECTION(); + } + /* Awaiting is the only way a coroutine body suspends. */ + gen->gi_frame_state = FRAME_SUSPENDED_YIELD_FROM; + _PyThreadState_PopFrame(tstate, frame); + return gen; +} + PyObject * _PyEval_GetAwaitable(PyObject *iterable, int oparg) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 36921e35533433..56d546dadd1680 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -156,6 +156,27 @@ # define LEAVE_TRACING() tracing_mode = 0 #endif +/* An awaited call (the low bit of oparg is set) of a coroutine function lets + * the body run in the pushed frame without creating the coroutine object + * until it first suspends (see RETURN_GENERATOR). Not while the JIT records + * a trace, which expects the regular path, nor while coroutine origin + * tracking is on, since the origin is captured when the object is created. */ +#if _Py_TIER2 +# define LAZY_CORO_ALLOWED() (!IS_JIT_TRACING()) +#else +# define LAZY_CORO_ALLOWED() 1 +#endif +#define MAYBE_START_LAZY_CORO(FRAME) \ + do { \ + _PyInterpreterFrame *lazy_frame_ = (FRAME); \ + if ((_PyFrame_GetCode(lazy_frame_)->co_flags & CO_COROUTINE) && \ + tstate->coroutine_origin_tracking_depth == 0 && \ + LAZY_CORO_ALLOWED()) \ + { \ + lazy_frame_->lazy_coro = 1; \ + } \ + } while (0) + #if _Py_TIER2 #define STOP_TRACING() \ do { \ diff --git a/Python/codegen.c b/Python/codegen.c index a975705576a167..f84187ab5ef8ab 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -224,7 +224,8 @@ static int codegen_call_helper_impl(compiler *c, location loc, int n, /* Args already pushed */ asdl_expr_seq *args, PyObject *injected_arg, - asdl_keyword_seq *keywords); + asdl_keyword_seq *keywords, + int awaited); static int codegen_call_helper(compiler *c, location loc, int n, asdl_expr_seq *args, asdl_keyword_seq *keywords); @@ -276,6 +277,10 @@ codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) return _PyInstructionSequence_Addop(seq, opcode, oparg_, loc); } +/* The oparg of CALL and CALL_KW: the number of arguments, with CALL_AWAITED + * set when the result is awaited right away. */ +#define CALL_OPARG(NARGS, AWAITED) ((NARGS) | ((AWAITED) ? CALL_AWAITED : 0)) + #define ADDOP_I(C, LOC, OP, O) \ RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) @@ -492,12 +497,14 @@ codegen_call_exit_with_nones(compiler *c, location loc) ADDOP_LOAD_CONST(c, loc, Py_None); ADDOP_LOAD_CONST(c, loc, Py_None); ADDOP_LOAD_CONST(c, loc, Py_None); - ADDOP_I(c, loc, CALL, 3); + ADDOP_I(c, loc, CALL, CALL_OPARG(3, 0)); return SUCCESS; } +/* If `yield` is a valid label, it is placed on the YIELD_VALUE. */ static int -codegen_add_yield_from(compiler *c, location loc, int await) +codegen_add_yield_from_impl(compiler *c, location loc, int await, + jump_target_label yield) { NEW_JUMP_TARGET_LABEL(c, send); NEW_JUMP_TARGET_LABEL(c, fail); @@ -508,6 +515,9 @@ codegen_add_yield_from(compiler *c, location loc, int await) // Set up a virtual try/except to handle when StopIteration is raised during // a close or throw call. The only way YIELD_VALUE raises if they do! ADDOP_JUMP(c, loc, SETUP_FINALLY, fail); + if (IS_JUMP_TARGET_LABEL(yield)) { + USE_LABEL(c, yield); + } ADDOP_I(c, loc, YIELD_VALUE, 1); ADDOP(c, NO_LOCATION, POP_BLOCK); ADDOP_I(c, loc, RESUME, await ? RESUME_AFTER_AWAIT : RESUME_AFTER_YIELD_FROM); @@ -521,6 +531,12 @@ codegen_add_yield_from(compiler *c, location loc, int await) return SUCCESS; } +static int +codegen_add_yield_from(compiler *c, location loc, int await) +{ + return codegen_add_yield_from_impl(c, loc, await, NO_LABEL); +} + static int codegen_pop_except_and_reraise(compiler *c, location loc) { @@ -1056,7 +1072,7 @@ codegen_apply_decorators(compiler *c, asdl_expr_seq* decos) for (Py_ssize_t i = asdl_seq_LEN(decos) - 1; i > -1; i--) { location loc = LOC((expr_ty)asdl_seq_GET(decos, i)); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); } return SUCCESS; } @@ -1584,11 +1600,11 @@ codegen_function(compiler *c, stmt_ty s, int is_async) RETURN_IF_ERROR(ret); if (num_typeparam_args > 0) { ADDOP_I(c, loc, SWAP, num_typeparam_args + 1); - ADDOP_I(c, loc, CALL, num_typeparam_args - 1); + ADDOP_I(c, loc, CALL, CALL_OPARG(num_typeparam_args - 1, 0)); } else { ADDOP(c, loc, PUSH_NULL); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); } } @@ -1758,7 +1774,7 @@ codegen_class(compiler *c, stmt_ty s) RETURN_IF_ERROR_IN_SCOPE(c, codegen_call_helper_impl(c, loc, 2, s->v.ClassDef.bases, &_Py_STR(generic_base), - s->v.ClassDef.keywords)); + s->v.ClassDef.keywords, 0)); PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 0); @@ -1770,7 +1786,7 @@ codegen_class(compiler *c, stmt_ty s) Py_DECREF(co); RETURN_IF_ERROR(ret); ADDOP(c, loc, PUSH_NULL); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); } else { RETURN_IF_ERROR(codegen_call_helper(c, loc, 2, s->v.ClassDef.bases, @@ -1859,7 +1875,7 @@ codegen_typealias(compiler *c, stmt_ty s) Py_DECREF(co); RETURN_IF_ERROR(ret); ADDOP(c, loc, PUSH_NULL); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); } RETURN_IF_ERROR(codegen_nameop(c, loc, name, Store)); return SUCCESS; @@ -4152,7 +4168,7 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) // Return 1 if the method call was optimized, 0 if not, and -1 on error. static int -maybe_optimize_method_call(compiler *c, expr_ty e) +maybe_optimize_method_call(compiler *c, expr_ty e, int awaited) { Py_ssize_t argsl, i, kwdsl; expr_ty meth = e->v.Call.func; @@ -4217,11 +4233,11 @@ maybe_optimize_method_call(compiler *c, expr_ty e) RETURN_IF_ERROR( codegen_call_simple_kw_helper(c, loc, kwds, kwdsl)); loc = update_start_location_to_match_attr(c, LOC(e), meth); - ADDOP_I(c, loc, CALL_KW, argsl + kwdsl); + ADDOP_I(c, loc, CALL_KW, CALL_OPARG(argsl + kwdsl, awaited)); } else { loc = update_start_location_to_match_attr(c, LOC(e), meth); - ADDOP_I(c, loc, CALL, argsl); + ADDOP_I(c, loc, CALL, CALL_OPARG(argsl, awaited)); } return 1; } @@ -4246,10 +4262,10 @@ codegen_validate_keywords(compiler *c, asdl_keyword_seq *keywords) } static int -codegen_call(compiler *c, expr_ty e) +codegen_call(compiler *c, expr_ty e, int awaited) { RETURN_IF_ERROR(codegen_validate_keywords(c, e->v.Call.keywords)); - int ret = maybe_optimize_method_call(c, e); + int ret = maybe_optimize_method_call(c, e, awaited); if (ret < 0) { return ERROR; } @@ -4263,13 +4279,39 @@ codegen_call(compiler *c, expr_ty e) location loc = LOC(e->v.Call.func); ADDOP(c, loc, PUSH_NULL); loc = LOC(e); - ret = codegen_call_helper(c, loc, 0, - e->v.Call.args, - e->v.Call.keywords); + ret = codegen_call_helper_impl(c, loc, 0, e->v.Call.args, NULL, + e->v.Call.keywords, awaited); USE_LABEL(c, skip_normal_call); return ret; } +/* `await ` marks the call as awaited and pushes a NULL `lazy_outcome` + * below the callable. If the call runs a coroutine body without creating the + * coroutine object, the body reports there how it ended, and END_AWAITED_CALL + * either skips the SEND loop or jumps to its YIELD_VALUE. */ +static int +codegen_await(compiler *c, expr_ty e) +{ + location loc = LOC(e); + expr_ty value = e->v.Await.value; + if (value->kind != Call_kind) { + VISIT(c, expr, value); + ADDOP_I(c, loc, GET_AWAITABLE, 0); + ADDOP(c, loc, PUSH_NULL); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADD_YIELD_FROM(c, loc, 1); + return SUCCESS; + } + NEW_JUMP_TARGET_LABEL(c, yield); + ADDOP(c, LOC(value), PUSH_NULL); + RETURN_IF_ERROR(codegen_call(c, value, 1)); + ADDOP_JUMP(c, loc, END_AWAITED_CALL, yield); + ADDOP_I(c, loc, GET_AWAITABLE, 0); + ADDOP(c, loc, PUSH_NULL); + ADDOP_LOAD_CONST(c, loc, Py_None); + return codegen_add_yield_from_impl(c, loc, 1, yield); +} + static int codegen_template_str(compiler *c, expr_ty e) { @@ -4327,7 +4369,7 @@ codegen_joined_str(compiler *c, expr_ty e) VISIT(c, expr, asdl_seq_GET(e->v.JoinedStr.values, i)); ADDOP_I(c, loc, LIST_APPEND, 1); } - ADDOP_I(c, loc, CALL, 1); + ADDOP_I(c, loc, CALL, CALL_OPARG(1, 0)); } else { VISIT_SEQ(c, expr, e->v.JoinedStr.values); @@ -4458,7 +4500,8 @@ codegen_call_helper_impl(compiler *c, location loc, int n, /* Args already pushed */ asdl_expr_seq *args, PyObject *injected_arg, - asdl_keyword_seq *keywords) + asdl_keyword_seq *keywords, + int awaited) { Py_ssize_t i, nseen, nelts, nkwelts; @@ -4497,10 +4540,10 @@ codegen_call_helper_impl(compiler *c, location loc, VISIT_SEQ(c, keyword, keywords); RETURN_IF_ERROR( codegen_call_simple_kw_helper(c, loc, keywords, nkwelts)); - ADDOP_I(c, loc, CALL_KW, n + nelts + nkwelts); + ADDOP_I(c, loc, CALL_KW, CALL_OPARG(n + nelts + nkwelts, awaited)); } else { - ADDOP_I(c, loc, CALL, n + nelts); + ADDOP_I(c, loc, CALL, CALL_OPARG(n + nelts, awaited)); } return SUCCESS; @@ -4556,7 +4599,7 @@ codegen_call_helper_impl(compiler *c, location loc, if (nkwelts == 0) { ADDOP(c, loc, PUSH_NULL); } - ADDOP(c, loc, CALL_FUNCTION_EX); + ADDOP_I(c, loc, CALL_FUNCTION_EX, awaited); return SUCCESS; } @@ -4566,7 +4609,7 @@ codegen_call_helper(compiler *c, location loc, asdl_expr_seq *args, asdl_keyword_seq *keywords) { - return codegen_call_helper_impl(c, loc, n, args, NULL, keywords); + return codegen_call_helper_impl(c, loc, n, args, NULL, keywords, 0); } /* List and set comprehensions work by being inlined at the location where @@ -5163,7 +5206,7 @@ codegen_comprehension(compiler *c, expr_ty e, int type, Py_CLEAR(co); VISIT(c, expr, outermost->iter); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); if (is_async_comprehension && type != COMP_GENEXP) { ADDOP_I(c, loc, GET_AWAITABLE, 0); @@ -5310,7 +5353,7 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) ADDOP_I(c, loc, SWAP, 2); ADDOP_I(c, loc, SWAP, 3); ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___AENTER__); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); ADDOP_I(c, loc, GET_AWAITABLE, 1); ADDOP(c, loc, PUSH_NULL); ADDOP_LOAD_CONST(c, loc, Py_None); @@ -5422,7 +5465,7 @@ codegen_with_inner(compiler *c, stmt_ty s, int pos) ADDOP_I(c, loc, SWAP, 2); ADDOP_I(c, loc, SWAP, 3); ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___ENTER__); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); ADDOP_JUMP(c, loc, SETUP_WITH, final); /* SETUP_WITH pushes a finally block. */ @@ -5550,16 +5593,11 @@ codegen_visit_expr_impl(compiler *c, expr_ty e, bool result_is_unused) ADD_YIELD_FROM(c, loc, 0); break; case Await_kind: - VISIT(c, expr, e->v.Await.value); - ADDOP_I(c, loc, GET_AWAITABLE, 0); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); - ADD_YIELD_FROM(c, loc, 1); - break; + return codegen_await(c, e); case Compare_kind: return codegen_compare(c, e); case Call_kind: - return codegen_call(c, e); + return codegen_call(c, e, 0); case Constant_kind: ADDOP_LOAD_CONST(c, loc, e->v.Constant.value); break; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 58473deddcebe3..7ffe6bd7dc3abd 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8940,6 +8940,168 @@ break; } + /* _END_AWAITED_CALL is not a viable micro-op for tier 2 because it is replaced */ + + case _GUARD_AWAITED_CALL_NOT_LAZY_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef lazy_outcome; + lazy_outcome = stack_pointer[-2]; + if (!PyStackRef_IsNull(lazy_outcome)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = lazy_outcome; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_AWAITED_CALL_NOT_LAZY_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef lazy_outcome; + _PyStackRef _stack_item_0 = _tos_cache0; + lazy_outcome = stack_pointer[-1]; + if (!PyStackRef_IsNull(lazy_outcome)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = lazy_outcome; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_AWAITED_CALL_NOT_LAZY_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef lazy_outcome; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + lazy_outcome = _stack_item_0; + if (!PyStackRef_IsNull(lazy_outcome)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = lazy_outcome; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = lazy_outcome; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_AWAITED_CALL_NOT_LAZY_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef lazy_outcome; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + lazy_outcome = _stack_item_1; + if (!PyStackRef_IsNull(lazy_outcome)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = lazy_outcome; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = lazy_outcome; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _POP_LAZY_OUTCOME_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef res; + _PyStackRef lazy_outcome; + _PyStackRef value; + res = stack_pointer[-1]; + lazy_outcome = stack_pointer[-2]; + assert(PyStackRef_IsNull(lazy_outcome)); + value = res; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _POP_LAZY_OUTCOME_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef res; + _PyStackRef lazy_outcome; + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + res = _stack_item_0; + lazy_outcome = stack_pointer[-1]; + assert(PyStackRef_IsNull(lazy_outcome)); + value = res; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _POP_LAZY_OUTCOME_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef res; + _PyStackRef lazy_outcome; + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + res = _stack_item_1; + lazy_outcome = _stack_item_0; + assert(PyStackRef_IsNull(lazy_outcome)); + value = res; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _POP_LAZY_OUTCOME_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef res; + _PyStackRef lazy_outcome; + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + res = _stack_item_2; + lazy_outcome = _stack_item_1; + assert(PyStackRef_IsNull(lazy_outcome)); + value = res; + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _SEND_GEN_FRAME_r33: { CHECK_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -9489,6 +9651,78 @@ break; } + /* _CHECK_LAZY_CORO is not a viable micro-op for tier 2 because it is replaced */ + + case _GUARD_NOT_LAZY_CORO_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + if (frame->lazy_coro) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOT_LAZY_CORO_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + if (frame->lazy_coro) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOT_LAZY_CORO_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + if (frame->lazy_coro) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOT_LAZY_CORO_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + if (frame->lazy_coro) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _YIELD_VALUE_r11: { CHECK_CURRENT_CACHED_VALUES(1); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -9499,34 +9733,48 @@ retval = _stack_item_0; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + int lazy = frame->lazy_coro; + PyGenObject *gen = lazy ? NULL : _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); + assert(!lazy || oparg == 1); _PyStackRef temp = retval; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); DTRACE_FUNCTION_RETURN(); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; + _PyErr_StackItem *exc_info = lazy ? tstate->exc_info : &gen->gi_exc_state; + tstate->exc_info = exc_info->previous_item; + exc_info->previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; - FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + if (!lazy) { + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + } assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_Wrap(gen_frame); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); _tos_cache0 = value; @@ -16860,8 +17108,8 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg & ~CALL_AWAITED)]; + callable = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -16869,8 +17117,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-1 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -16880,8 +17128,8 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-1 - (oparg & ~CALL_AWAITED)] = self_or_null; ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -17606,6 +17854,10 @@ break; } + /* _CLEAR_CALL_AWAITED is not a viable micro-op for tier 2 because it is replaced */ + + /* _START_LAZY_CORO is not a viable micro-op for tier 2 because it is replaced */ + case _GUARD_NOS_NULL_r02: { CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -19927,8 +20179,8 @@ _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg & ~CALL_AWAITED)]; + callable = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -19936,8 +20188,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-1 - (oparg & ~CALL_AWAITED)] = self_or_null; stack_pointer[0] = _stack_item_0; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -19951,8 +20203,8 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-1 - (oparg & ~CALL_AWAITED)] = self_or_null; ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -20720,33 +20972,58 @@ _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + _PyErr_StackItem *exc_info = NULL; + #if TIER_ONE + if (frame->lazy_coro) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + _PyFrame_StackPointerInvalidate(frame); + if (exc_info == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + } + } + #endif + if (exc_info != NULL) { + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + res = PyStackRef_None; + } + else { + frame->lazy_coro = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); _tos_cache0 = res; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -22851,6 +23128,8 @@ break; } + /* _MAYBE_LAZY_CORO is not a viable micro-op for tier 2 because it is replaced */ + case _SAVE_RETURN_OFFSET_r00: { CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); diff --git a/Python/flowgraph.c b/Python/flowgraph.c index fc676dc6feb167..354613f78bb71a 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -798,6 +798,12 @@ get_stack_effects(int opcode, int oparg, int jump, stack_effects *effects) effects->net = 0; return 0; } + if (opcode == END_AWAITED_CALL && jump) { + // The jump to YIELD_VALUE leaves (coroutine, NULL, value) in place + // of (lazy_outcome, value). + effects->net = 1; + return 0; + } effects->net = pushed - popped; return 0; } @@ -946,7 +952,12 @@ label_exception_targets(basicblock *entryblock) { else if (is_jump(instr)) { instr->i_except = handler; assert(i == b->b_iused -1); - if (!instr->i_target->b_visited) { + // END_AWAITED_CALL jumps to the YIELD_VALUE of the following + // SEND loop, inside the SETUP_FINALLY that the fallthrough + // from SEND sets up. + if (!instr->i_target->b_visited && + instr->i_opcode != END_AWAITED_CALL) + { if (BB_HAS_FALLTHROUGH(b)) { struct _PyCfgExceptStack *copy = copy_except_stack(except_stack); if (copy == NULL) { @@ -3117,6 +3128,14 @@ optimize_load_fast(cfg_builder *g) break; } + case END_AWAITED_CALL: { + load_fast_push_block(&sp, instr->i_target, refs.size + 1); + ref_stack_pop(&refs); + ref_stack_pop(&refs); + PUSH_REF(i, NOT_LOCAL); + break; + } + case SEND: { load_fast_push_block(&sp, instr->i_target, refs.size); ref_stack_pop(&refs); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e5b3548b66412c..8dac7c86b16929 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1831,8 +1831,8 @@ _PyStackRef res; // _SPECIALIZE_CALL { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg & ~CALL_AWAITED)]; + callable = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION @@ -1840,7 +1840,7 @@ next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, (oparg & ~CALL_AWAITED) + !PyStackRef_IsNull(self_or_null)); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } @@ -1858,8 +1858,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-1 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -1868,9 +1868,9 @@ } // _DO_CALL { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg & ~CALL_AWAITED)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = oparg & ~CALL_AWAITED; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -1882,8 +1882,8 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-1 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( @@ -1891,16 +1891,19 @@ arguments, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & CALL_AWAITED) && opcode != INSTRUMENTED_CALL) { + MAYBE_START_LAZY_CORO(new_frame); + } frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-1 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( @@ -1914,7 +1917,7 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -1922,8 +1925,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = res; + stack_pointer += -1 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2233,6 +2236,164 @@ DISPATCH(); } + TARGET(CALL_BOUND_METHOD_EXACT_ARGS_AWAITED) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BOUND_METHOD_EXACT_ARGS_AWAITED; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS_AWAITED); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + // _CLEAR_CALL_AWAITED + { + oparg &= ~CALL_AWAITED; + } + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_FUNCTION_VERSION + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _START_LAZY_CORO + { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(CALL_BOUND_METHOD_GENERAL) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_BOUND_METHOD_GENERAL; @@ -2369,6 +2530,150 @@ DISPATCH(); } + TARGET(CALL_BOUND_METHOD_GENERAL_AWAITED) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BOUND_METHOD_GENERAL_AWAITED; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL_AWAITED); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + // _CLEAR_CALL_AWAITED + { + oparg &= ~CALL_AWAITED; + } + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_METHOD_VERSION + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _EXPAND_METHOD + { + self_or_null = null; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _START_LAZY_CORO + { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(CALL_BUILTIN_CLASS) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_BUILTIN_CLASS; @@ -2972,6 +3277,13 @@ } ex_frame = PyStackRef_Wrap(new_frame); } + // _MAYBE_LAZY_CORO + { + new_frame = ex_frame; + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -2983,7 +3295,6 @@ } // _PUSH_FRAME { - new_frame = ex_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3150,6 +3461,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if (oparg & 1) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = 2u ; DISPATCH_INLINED(new_frame); @@ -3394,8 +3708,8 @@ _PyStackRef res; // _SPECIALIZE_CALL_KW { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + self_or_null = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; + callable = stack_pointer[-3 - (oparg & ~CALL_AWAITED)]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION @@ -3403,7 +3717,7 @@ next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_CallKw(callable, next_instr, (oparg & ~CALL_AWAITED) + !PyStackRef_IsNull(self_or_null)); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } @@ -3421,8 +3735,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -3432,10 +3746,10 @@ // _DO_CALL_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg & ~CALL_AWAITED)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; + int total_args = oparg & ~CALL_AWAITED; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3448,8 +3762,8 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( @@ -3457,7 +3771,7 @@ arguments, positional_args, kwnames_o, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -3466,47 +3780,189 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & CALL_AWAITED) && opcode != INSTRUMENTED_CALL_KW) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } + stack_pointer[-3 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( + callable, + arguments, + total_args, + kwnames, + opcode == INSTRUMENTED_CALL_KW, + frame, + this_instr, + tstate); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + stack_pointer += -3 - (oparg & ~CALL_AWAITED); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-3 - (oparg & ~CALL_AWAITED)] = res; + stack_pointer += -2 - (oparg & ~CALL_AWAITED); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CALL_KW_BOUND_METHOD) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_BOUND_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_METHOD_VERSION_KW + { + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _EXPAND_METHOD_KW + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + _PyStackRef callable_s = callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( - callable, - arguments, - total_args, - kwnames, - opcode == INSTRUMENTED_CALL_KW, - frame, - this_instr, - tstate); + PyStackRef_CLOSE(callable_s); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - if (res_o == NULL) { - stack_pointer += -3 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } - TARGET(CALL_KW_BOUND_METHOD) { + TARGET(CALL_KW_BOUND_METHOD_AWAITED) { #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW_BOUND_METHOD; + int opcode = CALL_KW_BOUND_METHOD_AWAITED; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD_AWAITED); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -3514,6 +3970,10 @@ _PyStackRef *args; _PyStackRef kwnames; _PyStackRef new_frame; + // _CLEAR_CALL_AWAITED + { + oparg &= ~CALL_AWAITED; + } /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -3605,6 +4065,10 @@ } new_frame = PyStackRef_Wrap(temp); } + // _START_LAZY_CORO + { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -3833,6 +4297,132 @@ DISPATCH(); } + TARGET(CALL_KW_PY_AWAITED) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_PY_AWAITED; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY_AWAITED); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + // _CLEAR_CALL_AWAITED + { + oparg &= ~CALL_AWAITED; + } + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _START_LAZY_CORO + { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(CALL_LEN) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_LEN; @@ -4590,39 +5180,162 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } - res = PyStackRef_FromPyObjectSteal(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_PY_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_PY_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); } - // _CHECK_PERIODIC_AT_END + // _SAVE_RETURN_OFFSET { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - int err = check_periodics_at_end(tstate, frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); } DISPATCH(); } - TARGET(CALL_PY_EXACT_ARGS) { + TARGET(CALL_PY_EXACT_ARGS_AWAITED) { #if _Py_TAIL_CALL_INTERP - int opcode = CALL_PY_EXACT_ARGS; + int opcode = CALL_PY_EXACT_ARGS_AWAITED; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS_AWAITED); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef new_frame; + // _CLEAR_CALL_AWAITED + { + oparg &= ~CALL_AWAITED; + } /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -4694,6 +5407,10 @@ } new_frame = PyStackRef_Wrap(pushed_frame); } + // _START_LAZY_CORO + { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -4834,6 +5551,121 @@ DISPATCH(); } + TARGET(CALL_PY_GENERAL_AWAITED) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_PY_GENERAL_AWAITED; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL_AWAITED); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + // _CLEAR_CALL_AWAITED + { + oparg &= ~CALL_AWAITED; + } + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _START_LAZY_CORO + { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(CALL_STR_1) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_STR_1; @@ -6119,6 +6951,74 @@ DISPATCH(); } + TARGET(END_AWAITED_CALL) { + #if _Py_TAIL_CALL_INTERP + int opcode = END_AWAITED_CALL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_AWAITED_CALL); + _PyStackRef lazy_outcome; + _PyStackRef res; + _PyStackRef value; + // _END_AWAITED_CALL + { + res = stack_pointer[-1]; + lazy_outcome = stack_pointer[-2]; + if (PyStackRef_IsTaggedInt(lazy_outcome)) { + lazy_outcome = PyStackRef_NULL; + _Py_CODEUNIT *send = next_instr + oparg - 1 - INLINE_CACHE_ENTRIES_SEND; + stack_pointer[-2] = lazy_outcome; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(_Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), + (int)(send - _PyFrame_GetBytecode(frame))).op.code == SEND); + _PyFrame_StackPointerInvalidate(frame); + _Py_CODEUNIT *end_send = next_instr + oparg + send->op.arg; + if (end_send->op.code != END_SEND) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = monitor_stop_iteration(tstate, frame, end_send, + PyStackRef_AsPyObjectBorrow(res)); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + JUMPBY(oparg + send->op.arg + 1); + } + else if (!PyStackRef_IsNull(lazy_outcome)) { + _PyInterpreterFrame *lazy_frame = PyStackRef_Unwrap(lazy_outcome); + lazy_outcome = PyStackRef_NULL; + stack_pointer[-2] = lazy_outcome; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = _PyEval_MaterializeLazyCoro(tstate, lazy_frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef v = res; + stack_pointer[-2] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + stack_pointer[-1] = PyStackRef_NULL; + stack_pointer[0] = v; + stack_pointer += 1; + JUMPBY(oparg); + DISPATCH(); + } + } + // _POP_LAZY_OUTCOME + { + assert(PyStackRef_IsNull(lazy_outcome)); + value = res; + } + stack_pointer[-2] = value; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + TARGET(END_FOR) { #if _Py_TAIL_CALL_INTERP int opcode = END_FOR; @@ -7111,8 +8011,8 @@ /* Skip 3 cache entries */ // _MAYBE_EXPAND_METHOD { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg & ~CALL_AWAITED)]; + callable = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -7120,8 +8020,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-1 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -7130,7 +8030,7 @@ } // _MONITOR_CALL { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg & ~CALL_AWAITED)]; maybe_self = self_or_null; func = callable; int is_meth = !PyStackRef_IsNull(maybe_self); @@ -7139,14 +8039,14 @@ if (is_meth) { arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); } - else if (oparg) { + else if (oparg & ~CALL_AWAITED) { arg0 = PyStackRef_AsPyObjectBorrow(args[0]); } else { arg0 = &_PyInstrumentation_MISSING; } - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = maybe_self; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = func; + stack_pointer[-1 - (oparg & ~CALL_AWAITED)] = maybe_self; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( @@ -7160,11 +8060,11 @@ } // _DO_CALL { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg & ~CALL_AWAITED)]; + self_or_null = stack_pointer[-1 - (oparg & ~CALL_AWAITED)]; + callable = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = oparg & ~CALL_AWAITED; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -7183,11 +8083,14 @@ arguments, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & CALL_AWAITED) && opcode != INSTRUMENTED_CALL) { + MAYBE_START_LAZY_CORO(new_frame); + } frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } @@ -7204,7 +8107,7 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -7212,8 +8115,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = res; + stack_pointer += -1 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -7355,6 +8258,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if (oparg & 1) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = 2u ; DISPATCH_INLINED(new_frame); @@ -7428,8 +8334,8 @@ /* Skip 2 cache entries */ // _MAYBE_EXPAND_METHOD_KW { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + self_or_null = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; + callable = stack_pointer[-3 - (oparg & ~CALL_AWAITED)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -7437,8 +8343,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -7447,7 +8353,7 @@ } // _MONITOR_CALL_KW { - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg & ~CALL_AWAITED)]; int is_meth = !PyStackRef_IsNull(self_or_null); PyObject *arg; if (is_meth) { @@ -7460,8 +8366,8 @@ arg = &_PyInstrumentation_MISSING; } PyObject *function = PyStackRef_AsPyObjectBorrow(callable); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( @@ -7475,10 +8381,10 @@ // _DO_CALL_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg & ~CALL_AWAITED)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; + int total_args = oparg & ~CALL_AWAITED; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -7498,7 +8404,7 @@ arguments, positional_args, kwnames_o, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -7507,6 +8413,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & CALL_AWAITED) && opcode != INSTRUMENTED_CALL_KW) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); @@ -7524,14 +8433,14 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-3 - (oparg & ~CALL_AWAITED)] = res; + stack_pointer += -2 - (oparg & ~CALL_AWAITED); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8282,6 +9191,9 @@ _PyStackRef val; _PyStackRef value; _PyStackRef retval; + // _CHECK_LAZY_CORO + { + } // _YIELD_VALUE_EVENT { val = stack_pointer[-1]; @@ -8315,36 +9227,50 @@ retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + int lazy = frame->lazy_coro; + PyGenObject *gen = lazy ? NULL : _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); + assert(!lazy || oparg == 1); _PyStackRef temp = retval; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); DTRACE_FUNCTION_RETURN(); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; + _PyErr_StackItem *exc_info = lazy ? tstate->exc_info : &gen->gi_exc_state; + tstate->exc_info = exc_info->previous_item; + exc_info->previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; - FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + if (!lazy) { + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + } assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_Wrap(gen_frame); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -11439,32 +12365,57 @@ _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - JUMP_TO_LABEL(error); + _PyErr_StackItem *exc_info = NULL; + #if TIER_ONE + if (frame->lazy_coro) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + _PyFrame_StackPointerInvalidate(frame); + if (exc_info == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + } + } + #endif + if (exc_info != NULL) { + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + res = PyStackRef_None; + } + else { + frame->lazy_coro = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -13571,6 +14522,9 @@ opcode = YIELD_VALUE; _PyStackRef value; _PyStackRef retval; + // _CHECK_LAZY_CORO + { + } // _MAKE_HEAP_SAFE { value = stack_pointer[-1]; @@ -13581,36 +14535,50 @@ retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + int lazy = frame->lazy_coro; + PyGenObject *gen = lazy ? NULL : _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); + assert(!lazy || oparg == 1); _PyStackRef temp = retval; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); DTRACE_FUNCTION_RETURN(); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; + _PyErr_StackItem *exc_info = lazy ? tstate->exc_info : &gen->gi_exc_state; + tstate->exc_info = exc_info->previous_item; + exc_info->previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; - FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + if (!lazy) { + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + } assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_Wrap(gen_frame); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -13763,6 +14731,9 @@ JUMP_TO_LABEL(error); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; + if (dying->lazy_coro) { + frame->instr_ptr += frame->return_offset; + } _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 3c9b2afcde100f..f02b9a511767a7 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -574,6 +574,7 @@ sanity_check_instrumentation(PyCodeObject *code) opcode = _PyCode_GetOriginalOpcode(data->lines, i); CHECK(valid_opcode(opcode)); CHECK(opcode != END_FOR); + CHECK(opcode != END_AWAITED_CALL); CHECK(opcode != RESUME); CHECK(opcode != RESUME_CHECK); CHECK(opcode != RESUME_CHECK_JIT); @@ -1574,9 +1575,12 @@ initialize_lines(_PyCoLineInstrumentationData *line_data, PyCodeObject *code, in case END_SEND: case RESUME: case POP_ITER: + case END_AWAITED_CALL: /* END_FOR cannot start a line, as it is skipped by FOR_ITER * END_SEND cannot start a line, as it is skipped by SEND - * RESUME and POP_ITER must not be instrumented with INSTRUMENTED_LINE */ + * RESUME and POP_ITER must not be instrumented with INSTRUMENTED_LINE + * END_AWAITED_CALL must run right after a lazily started + * coroutine suspends (see _YIELD_VALUE) */ set_original_opcode(line_data, i, 0); break; default: @@ -1901,7 +1905,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) if (removed_per_instruction_tools) { for (int i = code->_co_firsttraceable; i < code_len;) { int opcode = _Py_GetBaseCodeUnit(code, i).op.code; - if (opcode == RESUME || opcode == END_FOR) { + if (opcode == RESUME || opcode == END_FOR || opcode == END_AWAITED_CALL) { i += _PyInstruction_GetLength(code, i); continue; } @@ -1928,7 +1932,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) if (new_per_instruction_tools) { for (int i = code->_co_firsttraceable; i < code_len;) { int opcode = _Py_GetBaseCodeUnit(code, i).op.code; - if (opcode == RESUME || opcode == END_FOR) { + if (opcode == RESUME || opcode == END_FOR || opcode == END_AWAITED_CALL) { i += _PyInstruction_GetLength(code, i); continue; } diff --git a/Python/intrinsics.c b/Python/intrinsics.c index dfa81baa654e12..e3b0fd406a612a 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -144,7 +144,11 @@ static PyObject * stopiteration_error(PyThreadState* tstate, PyObject *exc) { _PyInterpreterFrame *frame = tstate->current_frame; - assert(frame->owner == FRAME_OWNED_BY_GENERATOR); + /* A coroutine that is awaited immediately runs in a thread-owned + * frame until it first suspends (see RETURN_GENERATOR). */ + assert(frame->owner == FRAME_OWNED_BY_GENERATOR || + (frame->owner == FRAME_OWNED_BY_THREAD && + (_PyFrame_GetCode(frame)->co_flags & CO_COROUTINE))); assert(PyExceptionInstance_Check(exc)); const char *msg = NULL; if (PyErr_GivenExceptionMatches(exc, PyExc_StopIteration)) { diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 91b424773224e4..b796c9b8af1b29 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -4,7 +4,6 @@ static void *opcode_targets_table[256] = { &&TARGET_BINARY_SLICE, &&TARGET_BUILD_TEMPLATE, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, - &&TARGET_CALL_FUNCTION_EX, &&TARGET_CHECK_EG_MATCH, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CLEANUP_THROW, @@ -17,8 +16,8 @@ static void *opcode_targets_table[256] = { &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, &&TARGET_GET_LEN, - &&TARGET_RESERVED, &&TARGET_INTERPRETER_EXIT, + &&TARGET_RESERVED, &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_LOCALS, &&TARGET_MAKE_FUNCTION, @@ -51,6 +50,7 @@ static void *opcode_targets_table[256] = { &&TARGET_BUILD_STRING, &&TARGET_BUILD_TUPLE, &&TARGET_CALL, + &&TARGET_CALL_FUNCTION_EX, &&TARGET_CALL_INTRINSIC_1, &&TARGET_CALL_INTRINSIC_2, &&TARGET_CALL_KW, @@ -64,6 +64,7 @@ static void *opcode_targets_table[256] = { &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, &&TARGET_END_ASYNC_FOR, + &&TARGET_END_AWAITED_CALL, &&TARGET_EXTENDED_ARG, &&TARGET_FOR_ITER, &&TARGET_GET_AWAITABLE, @@ -127,7 +128,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -146,7 +146,9 @@ static void *opcode_targets_table[256] = { &&TARGET_BINARY_OP_SUBTRACT_INT, &&TARGET_CALL_ALLOC_AND_ENTER_INIT, &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS_AWAITED, &&TARGET_CALL_BOUND_METHOD_GENERAL, + &&TARGET_CALL_BOUND_METHOD_GENERAL_AWAITED, &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_CALL_BUILTIN_FAST, &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, @@ -155,8 +157,10 @@ static void *opcode_targets_table[256] = { &&TARGET_CALL_EX_PY, &&TARGET_CALL_ISINSTANCE, &&TARGET_CALL_KW_BOUND_METHOD, + &&TARGET_CALL_KW_BOUND_METHOD_AWAITED, &&TARGET_CALL_KW_NON_PY, &&TARGET_CALL_KW_PY, + &&TARGET_CALL_KW_PY_AWAITED, &&TARGET_CALL_LEN, &&TARGET_CALL_LIST_APPEND, &&TARGET_CALL_METHOD_DESCRIPTOR_FAST, @@ -165,7 +169,9 @@ static void *opcode_targets_table[256] = { &&TARGET_CALL_METHOD_DESCRIPTOR_O, &&TARGET_CALL_NON_PY_GENERAL, &&TARGET_CALL_PY_EXACT_ARGS, + &&TARGET_CALL_PY_EXACT_ARGS_AWAITED, &&TARGET_CALL_PY_GENERAL, + &&TARGET_CALL_PY_GENERAL_AWAITED, &&TARGET_CALL_STR_1, &&TARGET_CALL_TUPLE_1, &&TARGET_CALL_TYPE_1, @@ -227,12 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -376,7 +376,7 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -478,12 +478,12 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -561,7 +561,9 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CACHE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS_AWAITED(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_GENERAL_AWAITED(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); @@ -574,8 +576,10 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAM static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_BOUND_METHOD_AWAITED(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_PY_AWAITED(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); @@ -584,7 +588,9 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_EXACT_ARGS_AWAITED(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_GENERAL_AWAITED(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); @@ -607,6 +613,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_AWAITED_CALL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); @@ -804,7 +811,9 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [CALL] = _TAIL_CALL_CALL, [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT, [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS, + [CALL_BOUND_METHOD_EXACT_ARGS_AWAITED] = _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS_AWAITED, [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_CALL_BOUND_METHOD_GENERAL, + [CALL_BOUND_METHOD_GENERAL_AWAITED] = _TAIL_CALL_CALL_BOUND_METHOD_GENERAL_AWAITED, [CALL_BUILTIN_CLASS] = _TAIL_CALL_CALL_BUILTIN_CLASS, [CALL_BUILTIN_FAST] = _TAIL_CALL_CALL_BUILTIN_FAST, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS, @@ -817,8 +826,10 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [CALL_ISINSTANCE] = _TAIL_CALL_CALL_ISINSTANCE, [CALL_KW] = _TAIL_CALL_CALL_KW, [CALL_KW_BOUND_METHOD] = _TAIL_CALL_CALL_KW_BOUND_METHOD, + [CALL_KW_BOUND_METHOD_AWAITED] = _TAIL_CALL_CALL_KW_BOUND_METHOD_AWAITED, [CALL_KW_NON_PY] = _TAIL_CALL_CALL_KW_NON_PY, [CALL_KW_PY] = _TAIL_CALL_CALL_KW_PY, + [CALL_KW_PY_AWAITED] = _TAIL_CALL_CALL_KW_PY_AWAITED, [CALL_LEN] = _TAIL_CALL_CALL_LEN, [CALL_LIST_APPEND] = _TAIL_CALL_CALL_LIST_APPEND, [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST, @@ -827,7 +838,9 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O, [CALL_NON_PY_GENERAL] = _TAIL_CALL_CALL_NON_PY_GENERAL, [CALL_PY_EXACT_ARGS] = _TAIL_CALL_CALL_PY_EXACT_ARGS, + [CALL_PY_EXACT_ARGS_AWAITED] = _TAIL_CALL_CALL_PY_EXACT_ARGS_AWAITED, [CALL_PY_GENERAL] = _TAIL_CALL_CALL_PY_GENERAL, + [CALL_PY_GENERAL_AWAITED] = _TAIL_CALL_CALL_PY_GENERAL_AWAITED, [CALL_STR_1] = _TAIL_CALL_CALL_STR_1, [CALL_TUPLE_1] = _TAIL_CALL_CALL_TUPLE_1, [CALL_TYPE_1] = _TAIL_CALL_CALL_TYPE_1, @@ -850,6 +863,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [DICT_MERGE] = _TAIL_CALL_DICT_MERGE, [DICT_UPDATE] = _TAIL_CALL_DICT_UPDATE, [END_ASYNC_FOR] = _TAIL_CALL_END_ASYNC_FOR, + [END_AWAITED_CALL] = _TAIL_CALL_END_AWAITED_CALL, [END_FOR] = _TAIL_CALL_END_FOR, [END_SEND] = _TAIL_CALL_END_SEND, [ENTER_EXECUTOR] = _TAIL_CALL_ENTER_EXECUTOR, @@ -1005,7 +1019,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1016,12 +1029,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, - [220] = _TAIL_CALL_UNKNOWN_OPCODE, - [221] = _TAIL_CALL_UNKNOWN_OPCODE, - [222] = _TAIL_CALL_UNKNOWN_OPCODE, - [223] = _TAIL_CALL_UNKNOWN_OPCODE, - [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, [226] = _TAIL_CALL_UNKNOWN_OPCODE, [227] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1062,7 +1069,9 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [CALL] = _TAIL_CALL_TRACE_RECORD, [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_TRACE_RECORD, [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_EXACT_ARGS_AWAITED] = _TAIL_CALL_TRACE_RECORD, [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_GENERAL_AWAITED] = _TAIL_CALL_TRACE_RECORD, [CALL_BUILTIN_CLASS] = _TAIL_CALL_TRACE_RECORD, [CALL_BUILTIN_FAST] = _TAIL_CALL_TRACE_RECORD, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD, @@ -1075,8 +1084,10 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [CALL_ISINSTANCE] = _TAIL_CALL_TRACE_RECORD, [CALL_KW] = _TAIL_CALL_TRACE_RECORD, [CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_BOUND_METHOD_AWAITED] = _TAIL_CALL_TRACE_RECORD, [CALL_KW_NON_PY] = _TAIL_CALL_TRACE_RECORD, [CALL_KW_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_PY_AWAITED] = _TAIL_CALL_TRACE_RECORD, [CALL_LEN] = _TAIL_CALL_TRACE_RECORD, [CALL_LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_TRACE_RECORD, @@ -1085,7 +1096,9 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_TRACE_RECORD, [CALL_NON_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, [CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_EXACT_ARGS_AWAITED] = _TAIL_CALL_TRACE_RECORD, [CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_GENERAL_AWAITED] = _TAIL_CALL_TRACE_RECORD, [CALL_STR_1] = _TAIL_CALL_TRACE_RECORD, [CALL_TUPLE_1] = _TAIL_CALL_TRACE_RECORD, [CALL_TYPE_1] = _TAIL_CALL_TRACE_RECORD, @@ -1108,6 +1121,7 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [DICT_MERGE] = _TAIL_CALL_TRACE_RECORD, [DICT_UPDATE] = _TAIL_CALL_TRACE_RECORD, [END_ASYNC_FOR] = _TAIL_CALL_TRACE_RECORD, + [END_AWAITED_CALL] = _TAIL_CALL_TRACE_RECORD, [END_FOR] = _TAIL_CALL_TRACE_RECORD, [END_SEND] = _TAIL_CALL_TRACE_RECORD, [ENTER_EXECUTOR] = _TAIL_CALL_TRACE_RECORD, @@ -1263,7 +1277,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1274,12 +1287,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, - [220] = _TAIL_CALL_UNKNOWN_OPCODE, - [221] = _TAIL_CALL_UNKNOWN_OPCODE, - [222] = _TAIL_CALL_UNKNOWN_OPCODE, - [223] = _TAIL_CALL_UNKNOWN_OPCODE, - [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, [226] = _TAIL_CALL_UNKNOWN_OPCODE, [227] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/optimizer.c b/Python/optimizer.c index e05adb344c8d06..f10f3319c8aa72 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -502,6 +502,11 @@ _PyUOp_Replacements[MAX_UOP_ID + 1] = { [_LOAD_BYTECODE] = _NOP, [_SEND_VIRTUAL] = _SEND_VIRTUAL_TIER_TWO, [_SEND_ASYNC_GEN] = _SEND_ASYNC_GEN_TIER_TWO, + [_END_AWAITED_CALL] = _GUARD_AWAITED_CALL_NOT_LAZY, + [_MAYBE_LAZY_CORO] = _NOP, + [_CLEAR_CALL_AWAITED] = _NOP, + [_START_LAZY_CORO] = _NOP, + [_CHECK_LAZY_CORO] = _GUARD_NOT_LAZY_CORO, }; static const uint8_t @@ -918,6 +923,10 @@ _PyJit_translate_single_bytecode_to_trace( goto unsupported; } assert(nuops > 0); + if (_PyOpcode_Deopt[opcode] == CALL || _PyOpcode_Deopt[opcode] == CALL_KW) { + // See _CLEAR_CALL_AWAITED. + oparg &= ~CALL_AWAITED; + } uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM uint32_t orig_target = target; int record_idx = 0; @@ -966,6 +975,10 @@ _PyJit_translate_single_bytecode_to_trace( target = next_inst; } } + else if (uop == _GUARD_AWAITED_CALL_NOT_LAZY || + uop == _NOP || uop == _GUARD_NOT_LAZY_CORO) { + // These don't jump. + } else { int extended_arg = orig_oparg > 255; uint32_t jump_target = next_inst + orig_oparg + extended_arg; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 6284885c428832..bbbaf44a3e62cd 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1283,7 +1283,7 @@ dummy_func(void) { } } - op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg & ~CALL_AWAITED] -- callable, self_or_null, args[oparg & ~CALL_AWAITED])) { (void)args; callable = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); @@ -2649,15 +2649,15 @@ dummy_func(void) { sym_set_recorded_value(value, (PyObject *)this_instr->operand0); } - op(_RECORD_CALLABLE, (func, self, args[oparg] -- func, self, args[oparg])) { + op(_RECORD_CALLABLE, (func, self, args[oparg & ~CALL_AWAITED] -- func, self, args[oparg & ~CALL_AWAITED])) { sym_set_recorded_value(func, (PyObject *)this_instr->operand0); } - op(_RECORD_CALLABLE_KW, (func, self, args[oparg], kwnames -- func, self, args[oparg], kwnames)) { + op(_RECORD_CALLABLE_KW, (func, self, args[oparg & ~CALL_AWAITED], kwnames -- func, self, args[oparg & ~CALL_AWAITED], kwnames)) { sym_set_recorded_value(func, (PyObject *)this_instr->operand0); } - op(_RECORD_BOUND_METHOD, (callable, self, args[oparg] -- callable, self, args[oparg])) { + op(_RECORD_BOUND_METHOD, (callable, self, args[oparg & ~CALL_AWAITED] -- callable, self, args[oparg & ~CALL_AWAITED])) { sym_set_recorded_value(callable, (PyObject *)this_instr->operand0); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9cd20c495db441..b1823724910d2f 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1893,6 +1893,22 @@ break; } + /* _END_AWAITED_CALL is not a viable micro-op for tier 2 */ + + case _GUARD_AWAITED_CALL_NOT_LAZY: { + break; + } + + case _POP_LAZY_OUTCOME: { + JitOptRef value; + value = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = value; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + case _SEND_GEN_FRAME: { JitOptRef v; JitOptRef receiver; @@ -1963,6 +1979,12 @@ break; } + /* _CHECK_LAZY_CORO is not a viable micro-op for tier 2 */ + + case _GUARD_NOT_LAZY_CORO: { + break; + } + case _YIELD_VALUE: { JitOptRef retval; JitOptRef value; @@ -4047,14 +4069,14 @@ JitOptRef *args; JitOptRef self_or_null; JitOptRef callable; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg & ~CALL_AWAITED)]; + self_or_null = stack_pointer[-1 - (oparg & ~CALL_AWAITED)]; + callable = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; (void)args; callable = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg & ~CALL_AWAITED)] = callable; + stack_pointer[-1 - (oparg & ~CALL_AWAITED)] = self_or_null; break; } @@ -4275,6 +4297,10 @@ break; } + /* _CLEAR_CALL_AWAITED is not a viable micro-op for tier 2 */ + + /* _START_LAZY_CORO is not a viable micro-op for tier 2 */ + case _GUARD_NOS_NULL: { JitOptRef null; null = stack_pointer[-2]; @@ -5581,6 +5607,8 @@ break; } + /* _MAYBE_LAZY_CORO is not a viable micro-op for tier 2 */ + case _SAVE_RETURN_OFFSET: { break; } @@ -5824,21 +5852,21 @@ case _RECORD_CALLABLE: { JitOptRef func; - func = stack_pointer[-2 - oparg]; + func = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; sym_set_recorded_value(func, (PyObject *)this_instr->operand0); break; } case _RECORD_CALLABLE_KW: { JitOptRef func; - func = stack_pointer[-3 - oparg]; + func = stack_pointer[-3 - (oparg & ~CALL_AWAITED)]; sym_set_recorded_value(func, (PyObject *)this_instr->operand0); break; } case _RECORD_BOUND_METHOD: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; sym_set_recorded_value(callable, (PyObject *)this_instr->operand0); break; } diff --git a/Python/record_functions.c.h b/Python/record_functions.c.h index 98abe3d0505e20..1bcfc30d43085a 100644 --- a/Python/record_functions.c.h +++ b/Python/record_functions.c.h @@ -71,21 +71,21 @@ void _PyOpcode_RecordFunction_4OS(_PyInterpreterFrame *frame, _PyStackRef *stack void _PyOpcode_RecordFunction_CALLABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) { _PyStackRef func; - func = stack_pointer[-2 - oparg]; + func = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; *recorded_value = (PyObject *)PyStackRef_AsPyObjectBorrow(func); Py_INCREF(*recorded_value); } void _PyOpcode_RecordFunction_CALLABLE_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) { _PyStackRef func; - func = stack_pointer[-3 - oparg]; + func = stack_pointer[-3 - (oparg & ~CALL_AWAITED)]; *recorded_value = (PyObject *)PyStackRef_AsPyObjectBorrow(func); Py_INCREF(*recorded_value); } void _PyOpcode_RecordFunction_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) { _PyStackRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg & ~CALL_AWAITED)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) == &PyMethod_Type) { *recorded_value = (PyObject *)callable_o; @@ -171,10 +171,14 @@ const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = { [LOAD_ATTR_METHOD_LAZY_DICT] = {1, {_RECORD_TOS_INDEX}}, [CALL] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_PY_GENERAL] = {1, {_RECORD_CALLABLE_INDEX}}, + [CALL_PY_GENERAL_AWAITED] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_BOUND_METHOD_GENERAL] = {1, {_RECORD_CALLABLE_INDEX}}, + [CALL_BOUND_METHOD_GENERAL_AWAITED] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_NON_PY_GENERAL] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_BOUND_METHOD_EXACT_ARGS] = {1, {_RECORD_CALLABLE_INDEX}}, + [CALL_BOUND_METHOD_EXACT_ARGS_AWAITED] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_PY_EXACT_ARGS] = {1, {_RECORD_CALLABLE_INDEX}}, + [CALL_PY_EXACT_ARGS_AWAITED] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_TYPE_1] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_STR_1] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_TUPLE_1] = {1, {_RECORD_CALLABLE_INDEX}}, @@ -191,7 +195,9 @@ const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = { [CALL_METHOD_DESCRIPTOR_NOARGS] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_METHOD_DESCRIPTOR_FAST] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_KW_PY] = {1, {_RECORD_CALLABLE_KW_INDEX}}, + [CALL_KW_PY_AWAITED] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_KW_BOUND_METHOD] = {1, {_RECORD_CALLABLE_KW_INDEX}}, + [CALL_KW_BOUND_METHOD_AWAITED] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_KW] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_KW_NON_PY] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_FUNCTION_EX] = {1, {_RECORD_4OS_INDEX}}, @@ -229,10 +235,14 @@ const _PyOpcodeRecordSlotMap _PyOpcode_RecordSlotMaps[256] = { [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = {1, 1, {0}}, [LOAD_ATTR_METHOD_LAZY_DICT] = {1, 1, {0}}, [CALL_PY_GENERAL] = {1, 0, {0}}, + [CALL_PY_GENERAL_AWAITED] = {1, 0, {0}}, [CALL_BOUND_METHOD_GENERAL] = {1, 1, {0}}, + [CALL_BOUND_METHOD_GENERAL_AWAITED] = {1, 1, {0}}, [CALL_NON_PY_GENERAL] = {1, 0, {0}}, [CALL_BOUND_METHOD_EXACT_ARGS] = {1, 1, {0}}, + [CALL_BOUND_METHOD_EXACT_ARGS_AWAITED] = {1, 1, {0}}, [CALL_PY_EXACT_ARGS] = {1, 0, {0}}, + [CALL_PY_EXACT_ARGS_AWAITED] = {1, 0, {0}}, [CALL_ALLOC_AND_ENTER_INIT] = {1, 0, {0}}, [CALL_BUILTIN_CLASS] = {1, 0, {0}}, [CALL_BUILTIN_O] = {1, 0, {0}}, @@ -242,7 +252,9 @@ const _PyOpcodeRecordSlotMap _PyOpcode_RecordSlotMaps[256] = { [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = {1, 0, {0}}, [CALL_METHOD_DESCRIPTOR_NOARGS] = {1, 0, {0}}, [CALL_KW_PY] = {1, 0, {0}}, + [CALL_KW_PY_AWAITED] = {1, 0, {0}}, [CALL_KW_BOUND_METHOD] = {1, 0, {0}}, + [CALL_KW_BOUND_METHOD_AWAITED] = {1, 0, {0}}, [CALL_EX_PY] = {1, 0, {0}}, [BINARY_OP] = {2, 2, {1, 0}}, }; diff --git a/Python/specialize.c b/Python/specialize.c index b5cf4ebc31142a..98d44ddd2c5906 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1782,7 +1782,17 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, } write_u32(cache->func_version, version); uint8_t opcode; - if (argcount == nargs + bound_method) { + if (instr->op.arg & CALL_AWAITED) { + if (argcount == nargs + bound_method) { + opcode = bound_method ? CALL_BOUND_METHOD_EXACT_ARGS_AWAITED + : CALL_PY_EXACT_ARGS_AWAITED; + } + else { + opcode = bound_method ? CALL_BOUND_METHOD_GENERAL_AWAITED + : CALL_PY_GENERAL_AWAITED; + } + } + else if (argcount == nargs + bound_method) { opcode = bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; } @@ -1820,7 +1830,13 @@ specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, return -1; } write_u32(cache->func_version, version); - specialize(instr, bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY); + if (instr->op.arg & CALL_AWAITED) { + specialize(instr, bound_method ? CALL_KW_BOUND_METHOD_AWAITED + : CALL_KW_PY_AWAITED); + } + else { + specialize(instr, bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY); + } return 0; } @@ -1876,6 +1892,14 @@ _Py_Specialize_Call(_PyStackRef callable_st, _PyStackRef self_or_null_st, _Py_CO assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL); assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL); + // Awaited calls (see CALL_AWAITED) have specialized forms only for + // Python functions. + bool awaited = instr->op.arg & CALL_AWAITED; + if (awaited && !PyFunction_Check(callable) && !PyMethod_Check(callable)) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OTHER); + unspecialize(instr); + return; + } int fail; if (PyCFunction_CheckExact(callable)) { fail = specialize_c_call(callable, instr, nargs); @@ -1918,6 +1942,14 @@ _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[CALL_KW] == INLINE_CACHE_ENTRIES_CALL_KW); assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL_KW); + // Awaited calls (see CALL_AWAITED) have specialized forms only for + // Python functions. + bool awaited = instr->op.arg & CALL_AWAITED; + if (awaited && !PyFunction_Check(callable) && !PyMethod_Check(callable)) { + SPECIALIZATION_FAIL(CALL_KW, SPEC_FAIL_OTHER); + unspecialize(instr); + return; + } int fail; if (PyFunction_Check(callable)) { fail = specialize_py_call_kw((PyFunctionObject *)callable, instr, nargs, false);